From 0f607bc2919db2cf7da8d293256191b0caf5601a Mon Sep 17 00:00:00 2001 From: Dave Bort Date: Mon, 23 Sep 2024 12:40:37 -0700 Subject: [PATCH] Move examples/mediatek out from under the torch namespace (#5478) Summary: The code under examples/... is a proxy for user code, and users should never declare code under the `torch::` or `executorch::` namespaces. Move this code under the `example::` namespace to make it more clear that users should use their own namespaces when writing code like this. Pull Request resolved: https://github.com/pytorch/executorch/pull/5478 Test Plan: - Built using the instructions at https://github.com/pytorch/executorch/blob/main/examples/mediatek/README.md Reviewed By: JacobSzwejbka, cccclai Differential Revision: D62992974 Pulled By: dbort fbshipit-source-id: b01f1b33d2853a0555ae19d79769a5bb6d0ba853 (cherry picked from commit 182f1382984feb074ce4f9b599778d05d4a9886a) --- .../llama_runner/FileMemMapper.h | 4 +-- .../llama_runner/LlamaConfig.h | 4 +-- .../llama_runner/LlamaModelChunk.cpp | 4 +-- .../llama_runner/LlamaModelChunk.h | 6 ++-- .../llama_runner/LlamaRuntime.cpp | 4 +-- .../llama_runner/LlamaRuntime.h | 4 +-- .../llama_runner/ModelChunk.cpp | 20 ++++++++--- .../executor_runner/llama_runner/ModelChunk.h | 6 ++-- .../llama_runner/MultiModelLoader.cpp | 4 +-- .../llama_runner/MultiModelLoader.h | 4 +-- .../executor_runner/llama_runner/Utils.h | 4 +-- .../llm_helper/include/llm_types.h | 4 +-- .../llm_helper/include/mask_builder.h | 4 +-- .../llm_helper/include/rotary_embedding.h | 4 +-- .../llm_helper/include/token_embedding.h | 4 +-- .../llama_runner/llm_helper/mask_builder.cpp | 4 +-- .../llm_helper/rotary_embedding.cpp | 4 +-- .../llm_helper/token_embedding.cpp | 4 +-- .../executor_runner/mtk_executor_runner.cpp | 20 ++++++++--- .../mtk_llama_executor_runner.cpp | 33 ++++++++++++------- .../mtk_oss_executor_runner.cpp | 23 ++++++++++--- 21 files changed, 106 insertions(+), 62 deletions(-) diff --git a/examples/mediatek/executor_runner/llama_runner/FileMemMapper.h b/examples/mediatek/executor_runner/llama_runner/FileMemMapper.h index 5fc09428db8..1382e14d3e1 100644 --- a/examples/mediatek/executor_runner/llama_runner/FileMemMapper.h +++ b/examples/mediatek/executor_runner/llama_runner/FileMemMapper.h @@ -15,7 +15,7 @@ #include #include -namespace torch::executor { +namespace example { class FileMemMapper { // Read-only mmap public: @@ -97,4 +97,4 @@ class FileMemMapper { // Read-only mmap size_t mSize = 0; }; -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/LlamaConfig.h b/examples/mediatek/executor_runner/llama_runner/LlamaConfig.h index 5465299b32d..f512d59b5c5 100644 --- a/examples/mediatek/executor_runner/llama_runner/LlamaConfig.h +++ b/examples/mediatek/executor_runner/llama_runner/LlamaConfig.h @@ -13,7 +13,7 @@ #include "llm_helper/include/llm_types.h" -namespace torch::executor { +namespace example { using llm_helper::LLMType; @@ -42,4 +42,4 @@ struct LlamaModelPaths { std::vector gen_model_paths; }; -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/LlamaModelChunk.cpp b/examples/mediatek/executor_runner/llama_runner/LlamaModelChunk.cpp index c2d75fd30ec..bf4b3eefdde 100644 --- a/examples/mediatek/executor_runner/llama_runner/LlamaModelChunk.cpp +++ b/examples/mediatek/executor_runner/llama_runner/LlamaModelChunk.cpp @@ -26,7 +26,7 @@ #include "llm_helper/include/mask_builder.h" #include "llm_helper/include/rotary_embedding.h" -namespace torch::executor { +namespace example { inline std::vector getIndexRange( const size_t startIndex, @@ -343,4 +343,4 @@ void LlamaModelChunk::InitCache() { } } -} // namespace torch::executor +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/LlamaModelChunk.h b/examples/mediatek/executor_runner/llama_runner/LlamaModelChunk.h index c8955378cbf..0a5002199db 100644 --- a/examples/mediatek/executor_runner/llama_runner/LlamaModelChunk.h +++ b/examples/mediatek/executor_runner/llama_runner/LlamaModelChunk.h @@ -27,12 +27,12 @@ #include "llm_helper/include/mask_builder.h" #include "llm_helper/include/rotary_embedding.h" -namespace torch::executor { +namespace example { using llm_helper::MaskBuilder; using llm_helper::RotaryEmbeddingMasterLut; -using TensorShape = Span; +using TensorShape = executorch::runtime::Span; using ModelIndexMap = std::unordered_map; // Llama decoder chunk @@ -135,4 +135,4 @@ class LlamaModelChunk : public ModelChunk { size_t mCurrentTokenIndex = 0; }; -} // namespace torch::executor +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/LlamaRuntime.cpp b/examples/mediatek/executor_runner/llama_runner/LlamaRuntime.cpp index 0d2d5ccd59c..2254241a001 100644 --- a/examples/mediatek/executor_runner/llama_runner/LlamaRuntime.cpp +++ b/examples/mediatek/executor_runner/llama_runner/LlamaRuntime.cpp @@ -18,7 +18,7 @@ #include "llm_helper/include/rotary_embedding.h" #include "llm_helper/include/token_embedding.h" -namespace torch::executor { +namespace example { void LlamaRuntime::Initialize( const LlamaModelOptions& modelOptions, @@ -201,4 +201,4 @@ const LlamaModelOptions& LlamaRuntime::GetModelOptions() const { return mModelOptions; } -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/LlamaRuntime.h b/examples/mediatek/executor_runner/llama_runner/LlamaRuntime.h index d788e73dcaa..fc2fca2e105 100644 --- a/examples/mediatek/executor_runner/llama_runner/LlamaRuntime.h +++ b/examples/mediatek/executor_runner/llama_runner/LlamaRuntime.h @@ -20,7 +20,7 @@ #include "llm_helper/include/rotary_embedding.h" #include "llm_helper/include/token_embedding.h" -namespace torch::executor { +namespace example { class LlamaRuntime { public: @@ -56,4 +56,4 @@ class LlamaRuntime { size_t mTokenIndex = 0; }; -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/ModelChunk.cpp b/examples/mediatek/executor_runner/llama_runner/ModelChunk.cpp index b09e2c58767..7f67fb4ca79 100644 --- a/examples/mediatek/executor_runner/llama_runner/ModelChunk.cpp +++ b/examples/mediatek/executor_runner/llama_runner/ModelChunk.cpp @@ -23,9 +23,21 @@ #define ENSURE_INIT \ ET_CHECK_MSG(Initialized(), "Error: Model chunk not initialized."); -namespace torch::executor { - -using util::FileDataLoader; +namespace example { + +using executorch::aten::Tensor; +using executorch::aten::TensorImpl; +using executorch::extension::FileDataLoader; +using executorch::runtime::Error; +using executorch::runtime::HierarchicalAllocator; +using executorch::runtime::MemoryAllocator; +using executorch::runtime::MemoryManager; +using executorch::runtime::Method; +using executorch::runtime::MethodMeta; +using executorch::runtime::Program; +using executorch::runtime::Result; +using executorch::runtime::Span; +using executorch::runtime::Tag; static constexpr size_t kMethodAllocatorPoolSize = 4 * 1024U * 1024U; // 4MB @@ -572,4 +584,4 @@ void ModelChunk::ReleaseModelInstance(void* modelInstance) { } } -} // namespace torch::executor +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/ModelChunk.h b/examples/mediatek/executor_runner/llama_runner/ModelChunk.h index 988747e47e3..67d9e30b5f1 100644 --- a/examples/mediatek/executor_runner/llama_runner/ModelChunk.h +++ b/examples/mediatek/executor_runner/llama_runner/ModelChunk.h @@ -16,7 +16,7 @@ #include "MultiModelLoader.h" -namespace torch::executor { +namespace example { struct BufferInfo { void* data = nullptr; @@ -91,7 +91,7 @@ class ModelChunk : protected MultiTokenSizeModelLoader { // Release allocated buffers for model IOs void ReleaseIoBuffers(); - Method& GetModelMethod(); + executorch::runtime::Method& GetModelMethod(); private: // Override the virtual functions @@ -119,4 +119,4 @@ class ModelChunk : protected MultiTokenSizeModelLoader { std::unordered_map mModelOutToInIndexLinks; }; -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/MultiModelLoader.cpp b/examples/mediatek/executor_runner/llama_runner/MultiModelLoader.cpp index e0479110a7c..4c33e174b74 100644 --- a/examples/mediatek/executor_runner/llama_runner/MultiModelLoader.cpp +++ b/examples/mediatek/executor_runner/llama_runner/MultiModelLoader.cpp @@ -16,7 +16,7 @@ #include #include -namespace torch::executor { +namespace example { template void MultiModelLoader::LoadModels() { @@ -174,4 +174,4 @@ std::string MultiModelLoader::GetIdString(const IdType& id) { template class MultiModelLoader; template class MultiModelLoader; -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/MultiModelLoader.h b/examples/mediatek/executor_runner/llama_runner/MultiModelLoader.h index 49a400f4477..7c364b60c03 100644 --- a/examples/mediatek/executor_runner/llama_runner/MultiModelLoader.h +++ b/examples/mediatek/executor_runner/llama_runner/MultiModelLoader.h @@ -12,7 +12,7 @@ #include #include -namespace torch::executor { +namespace example { template class MultiModelLoader { @@ -92,4 +92,4 @@ class MultiModelLoader { IdType mCurrentModelId = 0; }; -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/Utils.h b/examples/mediatek/executor_runner/llama_runner/Utils.h index 9aed7ec08d4..24e8a4d6e50 100644 --- a/examples/mediatek/executor_runner/llama_runner/Utils.h +++ b/examples/mediatek/executor_runner/llama_runner/Utils.h @@ -18,7 +18,7 @@ #include #include -namespace torch::executor { +namespace example { namespace utils { class Timer { @@ -113,4 +113,4 @@ static std::string to_string(const std::vector vec) { } } // namespace utils -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/llm_helper/include/llm_types.h b/examples/mediatek/executor_runner/llama_runner/llm_helper/include/llm_types.h index e4cb14a2c98..59290f820fc 100644 --- a/examples/mediatek/executor_runner/llama_runner/llm_helper/include/llm_types.h +++ b/examples/mediatek/executor_runner/llama_runner/llm_helper/include/llm_types.h @@ -11,7 +11,7 @@ #include #include -namespace torch::executor { +namespace example { namespace llm_helper { typedef enum { INT4, INT8, INT16, FP16, INT32, FP32, INVALID } LLMType; @@ -72,4 +72,4 @@ inline const char* getLLMTypeName(const LLMType llm_type) { } } // namespace llm_helper -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/llm_helper/include/mask_builder.h b/examples/mediatek/executor_runner/llama_runner/llm_helper/include/mask_builder.h index 14b40619ad3..5ab6741c11c 100644 --- a/examples/mediatek/executor_runner/llama_runner/llm_helper/include/mask_builder.h +++ b/examples/mediatek/executor_runner/llama_runner/llm_helper/include/mask_builder.h @@ -12,7 +12,7 @@ #include -namespace torch::executor { +namespace example { namespace llm_helper { class MaskBuilder { @@ -76,4 +76,4 @@ class MaskBuilder { }; } // namespace llm_helper -} // namespace torch::executor +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/llm_helper/include/rotary_embedding.h b/examples/mediatek/executor_runner/llama_runner/llm_helper/include/rotary_embedding.h index cef7ec09e2a..d4c017cf82b 100644 --- a/examples/mediatek/executor_runner/llama_runner/llm_helper/include/rotary_embedding.h +++ b/examples/mediatek/executor_runner/llama_runner/llm_helper/include/rotary_embedding.h @@ -13,7 +13,7 @@ #include #include -namespace torch::executor { +namespace example { namespace llm_helper { class RotaryEmbeddingMasterLut { @@ -77,4 +77,4 @@ class RotaryEmbeddingMasterLut { }; } // namespace llm_helper -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/llm_helper/include/token_embedding.h b/examples/mediatek/executor_runner/llama_runner/llm_helper/include/token_embedding.h index d3ed623f5f0..43d6413be66 100644 --- a/examples/mediatek/executor_runner/llama_runner/llm_helper/include/token_embedding.h +++ b/examples/mediatek/executor_runner/llama_runner/llm_helper/include/token_embedding.h @@ -13,7 +13,7 @@ #include #include -namespace torch::executor { +namespace example { class FileMemMapper; @@ -49,4 +49,4 @@ class TokenEmbeddingLut { }; } // namespace llm_helper -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/llm_helper/mask_builder.cpp b/examples/mediatek/executor_runner/llama_runner/llm_helper/mask_builder.cpp index 9a7dafb2b7f..e83e8b37082 100644 --- a/examples/mediatek/executor_runner/llama_runner/llm_helper/mask_builder.cpp +++ b/examples/mediatek/executor_runner/llama_runner/llm_helper/mask_builder.cpp @@ -11,7 +11,7 @@ #include #include -namespace torch::executor { +namespace example { namespace llm_helper { // Define mask values for different types @@ -260,4 +260,4 @@ bool MaskBuilder::adjustMaskForPadding(const size_t tokenBatchSize) { } } // namespace llm_helper -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/llm_helper/rotary_embedding.cpp b/examples/mediatek/executor_runner/llama_runner/llm_helper/rotary_embedding.cpp index 9015d875495..6f1a64bedbc 100644 --- a/examples/mediatek/executor_runner/llama_runner/llm_helper/rotary_embedding.cpp +++ b/examples/mediatek/executor_runner/llama_runner/llm_helper/rotary_embedding.cpp @@ -16,7 +16,7 @@ #include #include -namespace torch::executor { +namespace example { namespace llm_helper { RotaryEmbeddingMasterLut::RotaryEmbeddingMasterLut( @@ -394,4 +394,4 @@ size_t RotaryEmbeddingMasterLut::getRotEmbedLength() const { } } // namespace llm_helper -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/llama_runner/llm_helper/token_embedding.cpp b/examples/mediatek/executor_runner/llama_runner/llm_helper/token_embedding.cpp index 1e20cc22594..b69bb713083 100644 --- a/examples/mediatek/executor_runner/llama_runner/llm_helper/token_embedding.cpp +++ b/examples/mediatek/executor_runner/llama_runner/llm_helper/token_embedding.cpp @@ -19,7 +19,7 @@ namespace fs = std::filesystem; -namespace torch::executor { +namespace example { namespace llm_helper { TokenEmbeddingLut::TokenEmbeddingLut( @@ -90,4 +90,4 @@ void TokenEmbeddingLut::lookupEmbedding(const std::vector& tokens) { } } // namespace llm_helper -} // namespace torch::executor \ No newline at end of file +} // namespace example diff --git a/examples/mediatek/executor_runner/mtk_executor_runner.cpp b/examples/mediatek/executor_runner/mtk_executor_runner.cpp index a6ab4eedab2..1d9d5522161 100644 --- a/examples/mediatek/executor_runner/mtk_executor_runner.cpp +++ b/examples/mediatek/executor_runner/mtk_executor_runner.cpp @@ -41,11 +41,21 @@ DEFINE_string( "Model serialized in flatbuffer format."); DEFINE_int32(iteration, 1, "Iterations of inference."); -using namespace torch::executor; -using torch::executor::util::FileDataLoader; +using executorch::extension::FileDataLoader; +using executorch::extension::prepare_input_tensors; +using executorch::runtime::Error; +using executorch::runtime::EValue; +using executorch::runtime::HierarchicalAllocator; +using executorch::runtime::MemoryAllocator; +using executorch::runtime::MemoryManager; +using executorch::runtime::Method; +using executorch::runtime::MethodMeta; +using executorch::runtime::Program; +using executorch::runtime::Result; +using executorch::runtime::Span; int main(int argc, char** argv) { - runtime_init(); + executorch::runtime::runtime_init(); gflags::ParseCommandLineFlags(&argc, &argv, true); if (argc != 1) { @@ -158,7 +168,7 @@ int main(int argc, char** argv) { // Allocate input tensors and set all of their elements to 1. The `inputs` // variable owns the allocated memory and must live past the last call to // `execute()`. - auto inputs = util::prepare_input_tensors(*method); + auto inputs = prepare_input_tensors(*method); ET_CHECK_MSG( inputs.ok(), "Could not prepare inputs: 0x%" PRIx32, @@ -196,7 +206,7 @@ int main(int argc, char** argv) { status = method->get_outputs(outputs.data(), outputs.size()); ET_CHECK(status == Error::Ok); // Print the first and last 100 elements of long lists of scalars. - std::cout << torch::executor::util::evalue_edge_items(100); + std::cout << executorch::extension::evalue_edge_items(100); for (int i = 0; i < outputs.size(); ++i) { std::cout << "Output " << i << ": " << outputs[i] << std::endl; } diff --git a/examples/mediatek/executor_runner/mtk_llama_executor_runner.cpp b/examples/mediatek/executor_runner/mtk_llama_executor_runner.cpp index e8ffa253dca..2ebacec2c56 100644 --- a/examples/mediatek/executor_runner/mtk_llama_executor_runner.cpp +++ b/examples/mediatek/executor_runner/mtk_llama_executor_runner.cpp @@ -131,9 +131,19 @@ DEFINE_string(prompt_file, "", "File containing the prompt text."); static constexpr int8_t kAddBos = 1; static constexpr int8_t kAddEos = 0; -using namespace torch::executor; -using namespace torch::executor::llm_helper; -using torch::executor::utils::Timer; +using namespace example::llm_helper; +using example::LlamaModelOptions; +using example::LlamaModelPaths; +using example::LlamaRuntime; +using example::utils::argmax; +using example::utils::read_file; +using example::utils::split; +using example::utils::Timer; +using example::utils::to_string; +using executorch::extension::llm::BPETokenizer; +using executorch::extension::llm::Tokenizer; +using executorch::runtime::Error; +using executorch::runtime::Result; LlamaModelOptions get_model_options() { LlamaModelOptions options = { @@ -159,8 +169,8 @@ LlamaModelPaths get_model_paths() { LlamaModelPaths model_paths = { .tokenizer_path = FLAGS_tokenizer_path, .token_embedding_path = FLAGS_token_embedding_path, - .prompt_model_paths = utils::split(FLAGS_prompt_model_paths, ','), - .gen_model_paths = utils::split(FLAGS_gen_model_paths, ',')}; + .prompt_model_paths = split(FLAGS_prompt_model_paths, ','), + .gen_model_paths = split(FLAGS_gen_model_paths, ',')}; return model_paths; } @@ -211,8 +221,7 @@ Result digest_prompt( const auto vocab_size = tokenizer->vocab_size(); const auto logits_type = llama_runtime.GetModelOptions().model_output_type; - const auto first_output_token = - utils::argmax(logits_type, logits, vocab_size); + const auto first_output_token = argmax(logits_type, logits, vocab_size); return first_output_token; } @@ -259,7 +268,7 @@ Error gen_response( timer_gen_token.End(); prev_token = output_token; - output_token = utils::argmax(logits_type, logits, vocab_size); + output_token = argmax(logits_type, logits, vocab_size); full_response_tokens.push_back(output_token); // Stop when output is EOS @@ -279,7 +288,7 @@ Error gen_response( } std::cout << "\n\n[Generated Tokens]\n" - << utils::to_string(full_response_tokens) << std::endl; + << to_string(full_response_tokens) << std::endl; ET_LOG( Info, @@ -314,7 +323,7 @@ Error inference( std::unique_ptr load_tokenizer() { std::unique_ptr tokenizer; if (FLAGS_tokenizer_type == "bpe") { - tokenizer = std::make_unique(); + tokenizer = std::make_unique(); } else if (FLAGS_tokenizer_type == "tiktoken") { tokenizer = example::get_tiktoken_for_llama(); } @@ -325,7 +334,7 @@ std::unique_ptr load_tokenizer() { } int main(int argc, char** argv) { - runtime_init(); + executorch::runtime::runtime_init(); gflags::ParseCommandLineFlags(&argc, &argv, true); if (argc != 1) { @@ -364,7 +373,7 @@ int main(int argc, char** argv) { // Run model ET_CHECK_MSG(!FLAGS_prompt_file.empty(), "No prompt file provided."); - std::string prompt = utils::read_file(FLAGS_prompt_file); + std::string prompt = read_file(FLAGS_prompt_file); inference(llama_runtime, tokenizer, prompt); // Release model diff --git a/examples/mediatek/executor_runner/mtk_oss_executor_runner.cpp b/examples/mediatek/executor_runner/mtk_oss_executor_runner.cpp index 3a1ad1d863b..bfa8aef38f0 100755 --- a/examples/mediatek/executor_runner/mtk_oss_executor_runner.cpp +++ b/examples/mediatek/executor_runner/mtk_oss_executor_runner.cpp @@ -51,14 +51,27 @@ DEFINE_string( "outputs", "Model output folder. Default to 'outputs'"); -using namespace torch::executor; -using torch::executor::MemoryAllocator; -using torch::executor::util::BufferCleanup; -using torch::executor::util::FileDataLoader; +using executorch::aten::Tensor; +using executorch::aten::TensorImpl; +using executorch::extension::BufferCleanup; +using executorch::extension::FileDataLoader; +using executorch::runtime::Error; +using executorch::runtime::EValue; +using executorch::runtime::HierarchicalAllocator; +using executorch::runtime::MemoryAllocator; +using executorch::runtime::MemoryManager; +using executorch::runtime::Method; +using executorch::runtime::MethodMeta; +using executorch::runtime::Program; +using executorch::runtime::Result; +using executorch::runtime::Span; +using executorch::runtime::Tag; +using executorch::runtime::TensorInfo; + using namespace std::filesystem; int main(int argc, char** argv) { - runtime_init(); + executorch::runtime::runtime_init(); gflags::ParseCommandLineFlags(&argc, &argv, true); if (argc != 1) {