From 01cb9d3d769b0510439d6660c7db467d4ca439f3 Mon Sep 17 00:00:00 2001
From: neuropilot-captain <neuropilot@mediatek.com>
Date: Fri, 1 Aug 2025 16:41:39 +0800
Subject: [PATCH] Fix MTK build scripts

---
 backends/mediatek/scripts/mtk_build.sh            |  3 +++
 examples/mediatek/CMakeLists.txt                  | 15 ++++++++++++++-
 .../executor_runner/mtk_llama_executor_runner.cpp |  2 +-
 examples/mediatek/mtk_build_examples.sh           |  4 +++-
 4 files changed, 21 insertions(+), 3 deletions(-)

diff --git a/backends/mediatek/scripts/mtk_build.sh b/backends/mediatek/scripts/mtk_build.sh
index 512b2a573d2..2d9a2faa5c3 100755
--- a/backends/mediatek/scripts/mtk_build.sh
+++ b/backends/mediatek/scripts/mtk_build.sh
@@ -25,6 +25,9 @@ cmake -DCMAKE_INSTALL_PREFIX="${build_dir}" \
       -DANDROID_ABI=arm64-v8a \
       -DANDROID_NATIVE_API_LEVEL=26 \
       -DANDROID_PLATFORM=android-26 \
+      -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
+      -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+      -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
       -DEXECUTORCH_BUILD_NEURON=ON \
       -B"${build_dir}"
 
diff --git a/examples/mediatek/CMakeLists.txt b/examples/mediatek/CMakeLists.txt
index 1a6a5369a13..2e79130e5c6 100644
--- a/examples/mediatek/CMakeLists.txt
+++ b/examples/mediatek/CMakeLists.txt
@@ -29,7 +29,10 @@ endif()
 set(_common_compile_options -Wno-deprecated-declarations -fPIC)
 
 # Let files say "include <executorch/path/to/header.h>".
-set(_common_include_directories ${EXECUTORCH_ROOT}/..)
+set(_common_include_directories ${EXECUTORCH_ROOT}/..
+  ${EXECUTORCH_ROOT}/runtime/core/portable_type/c10
+  ${EXECUTORCH_ROOT}/extension/llm/tokenizers/include
+  ${EXECUTORCH_ROOT}/extension/llm/tokenizers/third-party/json/single_include)
 
 #
 # The `_<target>_srcs` lists are defined by including ${EXECUTORCH_SRCS_FILE}.
@@ -71,9 +74,13 @@ if(${ANDROID})
 
   target_link_libraries(
     mtk_executor_runner ${_executor_runner_libs} executorch neuron_backend
+    executorch_core
+    extension_evalue_util
+    extension_runner_util
     gflags
   )
   target_compile_options(mtk_executor_runner PUBLIC ${_common_compile_options})
+  add_compile_definitions(C10_USING_CUSTOM_GENERATED_MACROS)
 
   set(_mtk_oss_executor_runner__srcs ${_executor_runner__srcs})
   list(
@@ -104,6 +111,7 @@ if(${ANDROID})
 
   target_link_libraries(mtk_oss_executor_runner
       ${_executor_runner_libs}
+      extension_module
       executorch
       neuron_backend
       gflags
@@ -142,6 +150,8 @@ if(${ANDROID})
   target_include_directories(
     tokenizer PUBLIC ${_common_include_directories} ${THIRD_PARTY_ABSL_DIR}
                      ${THIRD_PARTY_RE2_DIR} ${LLAMA2_TOKENIZER_DIR}/include
+                     ${CMAKE_CURRENT_BINARY_DIR}/tokenizers/third-party/pcre2
+                     ${EXECUTORCH_ROOT}/extension/llm/tokenizers/include
   )
   target_link_libraries(tokenizer PRIVATE re2::re2)
   target_sources(
@@ -149,6 +159,9 @@ if(${ANDROID})
     PRIVATE
       ${LLAMA2_TOKENIZER_DIR}/src/tiktoken.cpp
       ${LLAMA2_TOKENIZER_DIR}/src/llama2c_tokenizer.cpp
+      ${LLAMA2_TOKENIZER_DIR}/src/regex.cpp
+      ${LLAMA2_TOKENIZER_DIR}/src/bpe_tokenizer_base.cpp
+      ${LLAMA2_TOKENIZER_DIR}/src/re2_regex.cpp
       ${CMAKE_CURRENT_SOURCE_DIR}/../models/llama/tokenizer/llama_tiktoken.cpp
   )
 
diff --git a/examples/mediatek/executor_runner/mtk_llama_executor_runner.cpp b/examples/mediatek/executor_runner/mtk_llama_executor_runner.cpp
index 012206e5142..733cc8c3465 100644
--- a/examples/mediatek/executor_runner/mtk_llama_executor_runner.cpp
+++ b/examples/mediatek/executor_runner/mtk_llama_executor_runner.cpp
@@ -285,7 +285,7 @@ Error inference(
 std::unique_ptr<Tokenizer> load_tokenizer() {
   std::unique_ptr<Tokenizer> tokenizer;
   if (FLAGS_tokenizer_type == "bpe") {
-    tokenizer = std::make_unique<BPETokenizer>();
+    tokenizer = std::make_unique<Llama2cTokenizer>();
   } else if (FLAGS_tokenizer_type == "tiktoken") {
     tokenizer = example::get_tiktoken_for_llama();
   }
diff --git a/examples/mediatek/mtk_build_examples.sh b/examples/mediatek/mtk_build_examples.sh
index 966093854e6..a46bdd3d1ce 100755
--- a/examples/mediatek/mtk_build_examples.sh
+++ b/examples/mediatek/mtk_build_examples.sh
@@ -28,6 +28,9 @@ main() {
           -DANDROID_ABI=arm64-v8a \
           -DANDROID_NATIVE_API_LEVEL=26 \
           -DANDROID_PLATFORM=android-26 \
+          -DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
+          -DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
+          -DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
           -DEXECUTORCH_BUILD_NEURON=ON \
           -B"${build_dir}"
 
@@ -48,7 +51,6 @@ main() {
           -DANDROID_NATIVE_API_LEVEL=26 \
           -DANDROID_PLATFORM=android-26 \
           -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=BOTH \
-          -DNEURON_BUFFER_ALLOCATOR_LIB="$NEURON_BUFFER_ALLOCATOR_LIB" \
           -B"${example_build_dir}" \
           $EXECUTORCH_ROOT/$example_dir