diff --git a/.github/workflows/_android.yml b/.github/workflows/_android.yml
index a38ed4527e9..2a91fb45ffa 100644
--- a/.github/workflows/_android.yml
+++ b/.github/workflows/_android.yml
@@ -131,7 +131,8 @@ jobs:
           # https://github.com/ReactiveCircus/android-emulator-runner. The max number
           # of cores we can set is 6, any higher number will be reduced to 6.
           cores: 6
-          ram-size: 12288M
+          ram-size: 16384M
+          heap-size: 12288M
           force-avd-creation: false
           disable-animations: true
           emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
diff --git a/extension/android/executorch_android/android_test_setup.sh b/extension/android/executorch_android/android_test_setup.sh
index 68f4d9dd6f6..fff8ba33618 100644
--- a/extension/android/executorch_android/android_test_setup.sh
+++ b/extension/android/executorch_android/android_test_setup.sh
@@ -12,17 +12,18 @@ if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
 fi
 which "${PYTHON_EXECUTABLE}"
 
-BASEDIR=$(dirname "$0")
+BASEDIR=$(dirname "$(realpath $0)")
 cp "${BASEDIR}/../../../extension/module/test/resources/add.pte" "${BASEDIR}/src/androidTest/resources"
 
 pushd "${BASEDIR}/../../../"
-curl -Ls "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories110M.pt" --output stories110M.pt
-curl -Ls "https://raw.githubusercontent.com/karpathy/llama2.c/master/tokenizer.model" --output tokenizer.model
+curl -C - -Ls "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories110M.pt" --output stories110M.pt
+curl -C - -Ls "https://raw.githubusercontent.com/karpathy/llama2.c/master/tokenizer.model" --output tokenizer.model
 # Create params.json file
 touch params.json
 echo '{"dim": 768, "multiple_of": 32, "n_heads": 12, "n_layers": 12, "norm_eps": 1e-05, "vocab_size": 32000}' > params.json
-python -m examples.models.llama.export_llama -c stories110M.pt -p params.json -X -kv --model=stories110m
+python -m examples.models.llama.export_llama -c stories110M.pt -p params.json -d fp16 -n stories110m_h.pte -kv
+python -m pytorch_tokenizers.tools.llama2c.convert -t tokenizer.model -o tokenizer.bin
 
-cp *.pte "${BASEDIR}/src/androidTest/resources/stories.pte"
-cp tokenizer.model "${BASEDIR}/src/androidTest/resources/tokenizer.bin"
+cp stories110m_h.pte "${BASEDIR}/src/androidTest/resources/stories.pte"
+cp tokenizer.bin "${BASEDIR}/src/androidTest/resources/tokenizer.bin"
 popd
diff --git a/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/LlmModuleInstrumentationTest.java b/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/LlmModuleInstrumentationTest.java
index ae81a21f420..e95b42f2650 100644
--- a/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/LlmModuleInstrumentationTest.java
+++ b/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/LlmModuleInstrumentationTest.java
@@ -86,7 +86,6 @@ public void testGenerate() throws IOException, URISyntaxException{
 
     @Test
     public void testGenerateAndStop() throws IOException, URISyntaxException{
-        int seqLen = 32;
         mModule.generate(TEST_PROMPT, SEQ_LEN, new LlmCallback() {
             @Override
             public void onResult(String result) {
diff --git a/scripts/run_android_emulator.sh b/scripts/run_android_emulator.sh
index fe73ec8a1d7..52f5fe55bd8 100755
--- a/scripts/run_android_emulator.sh
+++ b/scripts/run_android_emulator.sh
@@ -18,17 +18,14 @@ $ADB_PATH wait-for-device shell 'while [[ -z $(getprop sys.boot_completed) ]]; d
 echo "List all running emulators"
 $ADB_PATH devices
 
-adb uninstall com.example.executorchllamademo || true
-adb uninstall com.example.executorchllamademo.test || true
-adb install -t app-debug.apk
-adb install -t app-debug-androidTest.apk
-
-adb shell mkdir -p /data/local/tmp/llama
-adb push model.pte /data/local/tmp/llama
-adb push tokenizer.bin /data/local/tmp/llama
-adb shell am instrument -w -r com.example.executorchllamademo.test/androidx.test.runner.AndroidJUnitRunner
-
 adb uninstall org.pytorch.executorch.test || true
 adb install -t android-test-debug-androidTest.apk
 
-adb shell am instrument -w -r org.pytorch.executorch.test/androidx.test.runner.AndroidJUnitRunner
+adb logcat -c
+adb shell am instrument -w -r -e class org.pytorch.executorch.ModuleInstrumentationTest \
+  org.pytorch.executorch.test/androidx.test.runner.AndroidJUnitRunner >result.txt 2>&1
+adb logcat -d > logcat.txt
+cat logcat.txt
+grep -q FAILURES result.txt && cat result.txt
+grep -q FAILURES result.txt && exit -1
+exit 0