From e24529169af72955fccb4fea9c3296989f77ada5 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 12 Apr 2024 18:57:25 -0700 Subject: [PATCH 1/8] Try to revert part of #2962 To see if it fixes the Android app --- extension/android/CMakeLists.txt | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/extension/android/CMakeLists.txt b/extension/android/CMakeLists.txt index 4d4460203c0..c69cea0323e 100644 --- a/extension/android/CMakeLists.txt +++ b/extension/android/CMakeLists.txt @@ -58,9 +58,14 @@ if(EXECUTORCH_BUILD_LLAMA_JNI) add_library(llama_runner STATIC IMPORTED) set_property(TARGET llama_runner PROPERTY IMPORTED_LOCATION ${LLAMA_RUNNER_PATH}) + set(CUSTOM_OPS_LIB_PATH ${CMAKE_CURRENT_BINARY_DIR}/../../examples/models/llama2/custom_ops/libcustom_ops_lib.a) + add_library(custom_ops_lib STATIC IMPORTED) + set_property(TARGET custom_ops_lib PROPERTY IMPORTED_LOCATION ${CUSTOM_OPS_LIB_PATH}) + set(CUSTOM_OPS_PATH ${CMAKE_CURRENT_BINARY_DIR}/../../examples/models/llama2/custom_ops/libcustom_ops.a) add_library(custom_ops STATIC IMPORTED) set_property(TARGET custom_ops PROPERTY IMPORTED_LOCATION ${CUSTOM_OPS_PATH}) + target_link_options_shared_lib(custom_ops_lib) if(TARGET pthreadpool) set(LLAMA_JNI_SRCS jni/jni_layer_llama.cpp ../../backends/xnnpack/threadpool/cpuinfo_utils.cpp) @@ -77,6 +82,6 @@ if(EXECUTORCH_BUILD_LLAMA_JNI) endif() target_include_directories(executorch_llama_jni PRIVATE ${_common_include_directories}) target_link_libraries(executorch_llama_jni ${link_libraries} llama_runner - custom_ops cpublas eigen_blas) + custom_ops custom_ops_lib cpublas eigen_blas) target_compile_options(executorch_llama_jni PUBLIC ${_common_compile_options}) endif() From ef402a5c0538e42b09bd046770032874ae60c015 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 12 Apr 2024 18:59:45 -0700 Subject: [PATCH 2/8] Update Android triggers --- .github/workflows/android.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 6948f9fd1a0..ba7fb9140b6 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -10,7 +10,8 @@ on: - .ci/docker/** - .github/workflows/android.yml - install_requirements.sh - - examples/demo-apps/** + - examples/demo-apps/android/** + - extension/android/** - extension/module/** workflow_dispatch: From e5e911bbe240323ce4eb3002a747223e2e055dff Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 12 Apr 2024 20:55:12 -0700 Subject: [PATCH 3/8] Let's see if this works --- extension/android/CMakeLists.txt | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/extension/android/CMakeLists.txt b/extension/android/CMakeLists.txt index c69cea0323e..cc7afc0c80f 100644 --- a/extension/android/CMakeLists.txt +++ b/extension/android/CMakeLists.txt @@ -58,14 +58,10 @@ if(EXECUTORCH_BUILD_LLAMA_JNI) add_library(llama_runner STATIC IMPORTED) set_property(TARGET llama_runner PROPERTY IMPORTED_LOCATION ${LLAMA_RUNNER_PATH}) - set(CUSTOM_OPS_LIB_PATH ${CMAKE_CURRENT_BINARY_DIR}/../../examples/models/llama2/custom_ops/libcustom_ops_lib.a) - add_library(custom_ops_lib STATIC IMPORTED) - set_property(TARGET custom_ops_lib PROPERTY IMPORTED_LOCATION ${CUSTOM_OPS_LIB_PATH}) - set(CUSTOM_OPS_PATH ${CMAKE_CURRENT_BINARY_DIR}/../../examples/models/llama2/custom_ops/libcustom_ops.a) add_library(custom_ops STATIC IMPORTED) set_property(TARGET custom_ops PROPERTY IMPORTED_LOCATION ${CUSTOM_OPS_PATH}) - target_link_options_shared_lib(custom_ops_lib) + target_link_options_shared_lib(custom_ops) if(TARGET pthreadpool) set(LLAMA_JNI_SRCS jni/jni_layer_llama.cpp ../../backends/xnnpack/threadpool/cpuinfo_utils.cpp) @@ -82,6 +78,6 @@ if(EXECUTORCH_BUILD_LLAMA_JNI) endif() target_include_directories(executorch_llama_jni PRIVATE ${_common_include_directories}) target_link_libraries(executorch_llama_jni ${link_libraries} llama_runner - custom_ops custom_ops_lib cpublas eigen_blas) + custom_ops cpublas eigen_blas) target_compile_options(executorch_llama_jni PUBLIC ${_common_compile_options}) endif() From 3538293919ad5c359448b44cec45260434494ea5 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Sun, 14 Apr 2024 18:56:40 -0700 Subject: [PATCH 4/8] Update test spec --- .github/workflows/android.yml | 2 +- .../android-llama2-device-farm-test-spec.yml | 76 +++++++++++++++++++ 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index ba7fb9140b6..a4f25c9669c 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -102,7 +102,7 @@ jobs: android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/app-debug.apk android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/app-debug-androidTest.apk # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml - test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/414cb54d-4d83-4576-8317-93244e4dc50e + test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml # The exported llama2 model and its tokenizer, can be downloaded from https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b.zip. # Among the input, this is the biggest file and uploading it to AWS beforehand makes the test run much faster extra-data: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/bd15825b-ddab-4e47-9fef-a9c8935778dd diff --git a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml new file mode 100644 index 00000000000..57301b6fde0 --- /dev/null +++ b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml @@ -0,0 +1,76 @@ +version: 0.1 + +android_test_host: amazon_linux_2 + +phases: + install: + commands: + + pre_test: + commands: + # Prepare the model and the tokenizer + - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /sdcard/" + - adb -s $DEVICEFARM_DEVICE_UDID shell "mkdir -p /data/local/tmp/llama/" + - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/tokenizer.bin /data/local/tmp/llama/tokenizer.bin" + - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/xnnpack_llama2.pte /data/local/tmp/llama/xnnpack_llama2.pte" + - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/tokenizer.bin" + - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/xnnpack_llama2.pte" + - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /data/local/tmp/llama/" + + test: + commands: + # By default, the following ADB command is used by Device Farm to run your Instrumentation test. + # Please refer to Android's documentation for more options on running instrumentation tests with adb: + # https://developer.android.com/studio/test/command-line#run-tests-with-adb + - echo "Starting the Instrumentation test" + - | + adb -s $DEVICEFARM_DEVICE_UDID shell "am instrument -r -w --no-window-animation \ + $DEVICEFARM_TEST_PACKAGE_NAME/$DEVICEFARM_TEST_PACKAGE_RUNNER 2>&1 || echo \": -1\"" | + tee $DEVICEFARM_LOG_DIR/instrument.log + + # Parse the results + - |- + INSTRUMENT_LOG="$DEVICEFARM_LOG_DIR/instrument.log" + + DID_ANY_TESTS_START=$(grep "INSTRUMENTATION_STATUS_CODE: 1" $INSTRUMENT_LOG | wc -l); + TESTS_PASSED=$(grep "INSTRUMENTATION_STATUS_CODE: 0" $INSTRUMENT_LOG | wc -l); + TESTS_ERRORED=$(grep "INSTRUMENTATION_STATUS_CODE: -1" $INSTRUMENT_LOG | wc -l); + TESTS_FAILED=$(grep "INSTRUMENTATION_STATUS_CODE: -2" $INSTRUMENT_LOG | wc -l); + TESTS_IGNORED=$(grep "INSTRUMENTATION_STATUS_CODE: -3" $INSTRUMENT_LOG | wc -l); + TESTS_ASSUMPTION_FAILED=$(grep "INSTRUMENTATION_STATUS_CODE: -4" $INSTRUMENT_LOG | wc -l); + TESTS_PROCESSES_CRASHED=$(grep "INSTRUMENTATION_RESULT: shortMsg=Process crashed." $INSTRUMENT_LOG | wc -l); + + # And print the results so that the CI job can show them later + - | + INSTRUMENT_LOG="$DEVICEFARM_LOG_DIR/instrument.log" + + if [ $DID_ANY_TESTS_START -eq 0 ]; + then + echo "[PyTorch] Marking the test suite as failed because no tests started!"; + false; + elif [ $TESTS_FAILED -ne 0 ]; + then + OBSERVED_TPS=$(grep "The observed TPS " $INSTRUMENT_LOG | tail -n 1) + + if [ -n $OBSERVED_TPS ]; + then + echo "[PyTorch] ${OBSERVED_TPS}"; + else + echo "[PyTorch] Marking the test suite as failed because it failed to load the model"; + fi + elif [ $TESTS_ERRORED -ne 0 ]; + then + echo "[PyTorch] Marking the test suite as failed because $TESTS_ERRORED tests errored!"; + false; + elif [ $TESTS_PROCESSES_CRASHED -ne 0 ]; + then + echo "[PyTorch] Marking the test suite as failed because the app crashed due to OOM!"; + false; + fi; + + post_test: + commands: + +artifacts: + # By default, Device Farm will collect your artifacts from the $DEVICEFARM_LOG_DIR directory. + - $DEVICEFARM_LOG_DIR From 7c8e2721fdcb7acc82877f0bc859f17db1466167 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Sun, 14 Apr 2024 19:23:52 -0700 Subject: [PATCH 5/8] Use pre-built test spec --- .github/workflows/android.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index a4f25c9669c..be53f911635 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -102,7 +102,7 @@ jobs: android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/app-debug.apk android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/app-debug-androidTest.apk # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml - test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml + test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/e0e7318a-e3a2-4604-996c-f2762569c46b # The exported llama2 model and its tokenizer, can be downloaded from https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b.zip. # Among the input, this is the biggest file and uploading it to AWS beforehand makes the test run much faster extra-data: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/bd15825b-ddab-4e47-9fef-a9c8935778dd From 934bc83bffa622f248c55011379043c1bdb960b9 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Sun, 14 Apr 2024 19:49:21 -0700 Subject: [PATCH 6/8] Tweak the spec --- .github/workflows/android.yml | 2 +- .../android/LlamaDemo/android-llama2-device-farm-test-spec.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index be53f911635..a4f25c9669c 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -102,7 +102,7 @@ jobs: android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/app-debug.apk android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/app-debug-androidTest.apk # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml - test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/e0e7318a-e3a2-4604-996c-f2762569c46b + test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml # The exported llama2 model and its tokenizer, can be downloaded from https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b.zip. # Among the input, this is the biggest file and uploading it to AWS beforehand makes the test run much faster extra-data: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/bd15825b-ddab-4e47-9fef-a9c8935778dd diff --git a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml index 57301b6fde0..4df9f18cc5f 100644 --- a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml +++ b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml @@ -52,7 +52,7 @@ phases: then OBSERVED_TPS=$(grep "The observed TPS " $INSTRUMENT_LOG | tail -n 1) - if [ -n $OBSERVED_TPS ]; + if [ -n "${OBSERVED_TPS}" ]; then echo "[PyTorch] ${OBSERVED_TPS}"; else From e0fc351cf89df65eb22be2ac5bf018f336b8d725 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Sun, 14 Apr 2024 20:16:06 -0700 Subject: [PATCH 7/8] Use preset test spec --- .github/workflows/android.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index a4f25c9669c..9e2aa486007 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -102,7 +102,7 @@ jobs: android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/app-debug.apk android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/app-debug-androidTest.apk # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml - test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml + test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/8dfb4601-ea67-4f45-bdad-51af3c5e8a44 # The exported llama2 model and its tokenizer, can be downloaded from https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b.zip. # Among the input, this is the biggest file and uploading it to AWS beforehand makes the test run much faster extra-data: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/bd15825b-ddab-4e47-9fef-a9c8935778dd From f91c5d4355033b8277e15992284801c59c944bb4 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Sun, 14 Apr 2024 20:46:54 -0700 Subject: [PATCH 8/8] Update test spec --- .github/workflows/android.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 9e2aa486007..fa5cd854cc0 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -102,7 +102,7 @@ jobs: android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/app-debug.apk android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/app-debug-androidTest.apk # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml - test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/8dfb4601-ea67-4f45-bdad-51af3c5e8a44 + test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/abd86868-fa63-467e-a5c7-218194665a77 # The exported llama2 model and its tokenizer, can be downloaded from https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b.zip. # Among the input, this is the biggest file and uploading it to AWS beforehand makes the test run much faster extra-data: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/bd15825b-ddab-4e47-9fef-a9c8935778dd