diff --git a/.ci/scripts/build_android_instrumentation.sh b/.ci/scripts/build_android_instrumentation.sh
deleted file mode 100644
index 747dd7063a2..00000000000
--- a/.ci/scripts/build_android_instrumentation.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-# Copyright (c) Meta Platforms, Inc. and affiliates.
-# All rights reserved.
-#
-# This source code is licensed under the BSD-style license found in the
-# LICENSE file in the root directory of this source tree.
-
-set -ex
-
-if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
-  PYTHON_EXECUTABLE=python3
-fi
-which "${PYTHON_EXECUTABLE}"
-
-mkdir -p extension/android/executorch_android/src/androidTest/resources
-cp extension/module/test/resources/add.pte extension/android/executorch_android/src/androidTest/resources
-
-pushd extension/android
-ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew :executorch_android:testDebugUnitTest
-ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew :executorch_android:assembleAndroidTest
-popd
diff --git a/.github/workflows/_android.yml b/.github/workflows/_android.yml
index 6598b6f3b42..8378d7ed4d9 100644
--- a/.github/workflows/_android.yml
+++ b/.github/workflows/_android.yml
@@ -36,7 +36,8 @@ jobs:
         cp ${BUILD_AAR_DIR}/executorch.aar $ARTIFACTS_DIR_NAME
 
         mkdir -p ${ARTIFACTS_DIR_NAME}/library_test_dir
-        bash .ci/scripts/build_android_instrumentation.sh
+        bash extension/android/executorch_android/android_test_setup.sh
+        (cd extension/android; ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew :executorch_android:assembleAndroidTest)
         cp extension/android/executorch_android/build/outputs/apk/androidTest/debug/executorch_android-debug-androidTest.apk "${ARTIFACTS_DIR_NAME}/library_test_dir"
 
         mkdir -p ${ARTIFACTS_DIR_NAME}/fp32-xnnpack-custom
diff --git a/extension/android/README.md b/extension/android/README.md
new file mode 100644
index 00000000000..8972e615173
--- /dev/null
+++ b/extension/android/README.md
@@ -0,0 +1,46 @@
+# ExecuTorch Android
+
+This directory contains the Android Java/Kotlin binding. The final product is an AAR,
+which contains the `.so` libraries for c++ runtime, and `.jar` for Java API, and required
+metadata `AndroidManifest.xml`.
+
+## Core contents
+
+Under `extension/android/`,
+
+- `executorch_android/` is the root for the Java `org.pytorch.executorch` package
+  - `src/`
+    - `androidTest/` contains the android instrumentation test source
+    - `main/` contains the Java source
+    - `test/` contains the Java unit test source
+  - `build.gradle` is the rule to build the Java package.
+- `jni/` contains the JNI layer code, which depends on the ExecuTorch c++ runtime library.
+- `CMakeLists.txt` is the rule for building the JNI library.
+
+## Build
+
+`scripts/build_android_library.sh` is a helper script to build the Java library (into .jar), native library (into .so), and the packaged AAR file.
+
+The usage is:
+```sh
+export ANDROID_HOME=/path/to/sdk
+export ANDROID_NDK=/path/to/ndk
+sh scripts/build_android_library.sh
+```
+
+Please see [Android building from source](https://pytorch.org/executorch/main/using-executorch-android.html#building-from-source) for details
+
+## Test
+
+After the library is built,
+
+```sh
+# Set up models for testing
+sh executorch_android/android_test_setup.sh
+
+# Run unit test
+./gradlew :executorch_android:testDebugUnitTest
+
+# Run instrumentation test
+./gradlew :executorch_android:connectedAndroidTest
+```
diff --git a/extension/android/executorch_android/android_test_setup.sh b/extension/android/executorch_android/android_test_setup.sh
new file mode 100644
index 00000000000..68f4d9dd6f6
--- /dev/null
+++ b/extension/android/executorch_android/android_test_setup.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+# Copyright (c) Meta Platforms, Inc. and affiliates.
+# All rights reserved.
+#
+# This source code is licensed under the BSD-style license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -ex
+
+if [[ -z "${PYTHON_EXECUTABLE:-}" ]]; then
+  PYTHON_EXECUTABLE=python3
+fi
+which "${PYTHON_EXECUTABLE}"
+
+BASEDIR=$(dirname "$0")
+cp "${BASEDIR}/../../../extension/module/test/resources/add.pte" "${BASEDIR}/src/androidTest/resources"
+
+pushd "${BASEDIR}/../../../"
+curl -Ls "https://huggingface.co/karpathy/tinyllamas/resolve/main/stories110M.pt" --output stories110M.pt
+curl -Ls "https://raw.githubusercontent.com/karpathy/llama2.c/master/tokenizer.model" --output tokenizer.model
+# Create params.json file
+touch params.json
+echo '{"dim": 768, "multiple_of": 32, "n_heads": 12, "n_layers": 12, "norm_eps": 1e-05, "vocab_size": 32000}' > params.json
+python -m examples.models.llama.export_llama -c stories110M.pt -p params.json -X -kv --model=stories110m
+
+cp *.pte "${BASEDIR}/src/androidTest/resources/stories.pte"
+cp tokenizer.model "${BASEDIR}/src/androidTest/resources/tokenizer.bin"
+popd
diff --git a/extension/android/executorch_android/build.gradle b/extension/android/executorch_android/build.gradle
index 70e18289442..b1bc090759a 100644
--- a/extension/android/executorch_android/build.gradle
+++ b/extension/android/executorch_android/build.gradle
@@ -36,6 +36,10 @@ android {
     }
 }
 
+task copyTestRes(type: Exec) {
+  commandLine 'bash', 'android_test_setup.sh'
+}
+
 dependencies {
     implementation 'com.facebook.fbjni:fbjni:0.5.1'
     implementation 'com.facebook.soloader:nativeloader:0.10.5'
diff --git a/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/LlmModuleInstrumentationTest.java b/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/LlmModuleInstrumentationTest.java
index b3b515d7ed0..ae81a21f420 100644
--- a/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/LlmModuleInstrumentationTest.java
+++ b/extension/android/executorch_android/src/androidTest/java/org/pytorch/executorch/LlmModuleInstrumentationTest.java
@@ -40,7 +40,7 @@
 /** Unit tests for {@link org.pytorch.executorch.extension.llm.LlmModule}. */
 @RunWith(AndroidJUnit4.class)
 public class LlmModuleInstrumentationTest implements LlmCallback {
-    private static String TEST_FILE_NAME = "/tinyllama_portable_fp16_h.pte";
+    private static String TEST_FILE_NAME = "/stories.pte";
     private static String TOKENIZER_FILE_NAME = "/tokenizer.bin";
     private static String TEST_PROMPT = "Hello";
     private static int OK = 0x00;
diff --git a/scripts/build_android_library.sh b/scripts/build_android_library.sh
index 20450218de6..a32ac5f4e5b 100755
--- a/scripts/build_android_library.sh
+++ b/scripts/build_android_library.sh
@@ -119,6 +119,8 @@ build_aar() {
   fi
   pushd extension/android/
   ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew build
+  # Use java unit test as sanity check
+  ANDROID_HOME="${ANDROID_SDK:-/opt/android/sdk}" ./gradlew :executorch_android:testDebugUnitTest
   popd
   cp extension/android/executorch_android/build/outputs/aar/executorch_android-debug.aar "${BUILD_AAR_DIR}/executorch.aar"
 }