From d87872bcfd0d8ac79ce3423de6bb6b83b0390b69 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Fri, 9 Aug 2024 18:55:29 -0700
Subject: [PATCH 01/17] Not hardcode llama2 model in perf test

---
 .../android-llama2-device-farm-test-spec.yml  |  9 +++---
 .../example/executorchllamademo/PerfTest.java | 32 +++++++++++--------
 2 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
index 4df9f18cc5f..48e0b9e95f5 100644
--- a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
+++ b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
@@ -11,10 +11,10 @@ phases:
       # Prepare the model and the tokenizer
       - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /sdcard/"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "mkdir -p /data/local/tmp/llama/"
-      - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/tokenizer.bin /data/local/tmp/llama/tokenizer.bin"
-      - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/xnnpack_llama2.pte /data/local/tmp/llama/xnnpack_llama2.pte"
-      - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/tokenizer.bin"
-      - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/xnnpack_llama2.pte"
+      - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.bin /data/local/tmp/llama/"
+      - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.pte /data/local/tmp/llama/"
+      - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/*.bin"
+      - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/*.pte"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /data/local/tmp/llama/"
 
   test:
@@ -57,6 +57,7 @@ phases:
             echo "[PyTorch] ${OBSERVED_TPS}";
           else
             echo "[PyTorch] Marking the test suite as failed because it failed to load the model";
+            false;
           fi
         elif [ $TESTS_ERRORED -ne 0 ];
         then
diff --git a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
index b8988d1f4ba..df1912d9fcd 100644
--- a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
+++ b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
@@ -13,7 +13,9 @@
 import static org.junit.Assert.assertFalse;
 
 import androidx.test.ext.junit.runners.AndroidJUnit4;
+import java.io.File;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.List;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -24,7 +26,6 @@
 public class PerfTest implements LlamaCallback {
 
   private static final String RESOURCE_PATH = "/data/local/tmp/llama/";
-  private static final String MODEL_NAME = "xnnpack_llama2.pte";
   private static final String TOKENIZER_BIN = "tokenizer.bin";
 
   // From https://github.com/pytorch/executorch/blob/main/examples/models/llama2/README.md
@@ -35,22 +36,27 @@ public class PerfTest implements LlamaCallback {
 
   @Test
   public void testTokensPerSecond() {
-    String modelPath = RESOURCE_PATH + MODEL_NAME;
     String tokenizerPath = RESOURCE_PATH + TOKENIZER_BIN;
-    LlamaModule mModule = new LlamaModule(modelPath, tokenizerPath, 0.8f);
+    // Find out the model name
+    File directory = new File(RESOURCE_PATH);
+    File[] files = Arrays.stream(directory.listFiles())
+            .filter(file -> file.getName().endsWith(".pte"))
+            .forEach(model -> {
+              LlamaModule mModule = new LlamaModule(model.getPath(), tokenizerPath, 0.8f);
 
-    int loadResult = mModule.load();
-    // Check that the model can be load successfully
-    assertEquals(0, loadResult);
+              int loadResult = mModule.load();
+              // Check that the model can be load successfully
+              assertEquals(0, loadResult);
 
-    // Run a testing prompt
-    mModule.generate("How do you do! I'm testing llama2 on mobile device", PerfTest.this);
-    assertFalse(tokensPerSecond.isEmpty());
+              // Run a testing prompt
+              mModule.generate("How do you do! I'm testing llama2 on mobile device", PerfTest.this);
+              assertFalse(tokensPerSecond.isEmpty());
 
-    final Float tps = tokensPerSecond.get(tokensPerSecond.size() - 1);
-    assertTrue(
-        "The observed TPS " + tps + " is less than the expected TPS " + EXPECTED_TPS,
-        tps >= EXPECTED_TPS);
+              final Float tps = tokensPerSecond.get(tokensPerSecond.size() - 1);
+              // Just a hack to print the observed TPS for the model
+              System.out.println("The observed TPS for " + model.getName() + " is " + tps);
+              System.out.flush();
+            });
   }
 
   @Override

From 820b50ba81fb07a31563a58e2969bad0cbdfdc6d Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Fri, 9 Aug 2024 18:59:03 -0700
Subject: [PATCH 02/17] Also include the new spec

---
 .github/workflows/android-perf.yml | 4 ++--
 .github/workflows/android.yml      | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index a8223eef2c0..c93c3d1e848 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -218,7 +218,7 @@ jobs:
       # TODO: Hard code llm_demo_bpe for now in this job.
       android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_bpe/app-debug.apk
       android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_bpe/app-debug-androidTest.apk
-      # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml
-      test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/abd86868-fa63-467e-a5c7-218194665a77
+      # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
+      test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
       # Uploaded to S3 from the previous job
       extra-data: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/${{ matrix.model }}_${{ matrix.delegate }}/model.zip
diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml
index 7b3d8ab9a89..8d3440b7c15 100644
--- a/.github/workflows/android.yml
+++ b/.github/workflows/android.yml
@@ -170,8 +170,8 @@ jobs:
       # Uploaded to S3 from the previous job, the name of the app comes from the project itself
       android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug.apk
       android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug-androidTest.apk
-      # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec.yml
-      test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/abd86868-fa63-467e-a5c7-218194665a77
+      # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
+      test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
       # Among the input, this is the biggest file, so it is cached on AWS to make the test faster. Note that the file is deleted by AWS after 30
       # days and the job will automatically re-upload the file when that happens.
       extra-data: https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b-0717.zip

From b64f631ba564959c591a72ae55261470074652e6 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Fri, 9 Aug 2024 21:45:44 -0700
Subject: [PATCH 03/17] Debug

---
 .../LlamaDemo/android-llama2-device-farm-test-spec.yml   | 9 ++++++---
 .../java/com/example/executorchllamademo/PerfTest.java   | 4 ++++
 2 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
index 48e0b9e95f5..053ca171c78 100644
--- a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
+++ b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
@@ -48,7 +48,7 @@ phases:
         then
           echo "[PyTorch] Marking the test suite as failed because no tests started!";
           false;
-        elif [ $TESTS_FAILED -ne 0 ];
+        elif [ $TESTS_PASSED -ne 0 ];
         then
           OBSERVED_TPS=$(grep "The observed TPS " $INSTRUMENT_LOG | tail -n 1)
 
@@ -56,9 +56,12 @@ phases:
           then
             echo "[PyTorch] ${OBSERVED_TPS}";
           else
-            echo "[PyTorch] Marking the test suite as failed because it failed to load the model";
-            false;
+            echo "[PyTorch] Test passes but couldn't find the observed TPS from instrument log";
           fi
+        elif [ $TESTS_FAILED -ne 0 ];
+        then
+          echo "[PyTorch] Marking the test suite as failed because it failed to load the model";
+          false;
         elif [ $TESTS_ERRORED -ne 0 ];
         then
           echo "[PyTorch] Marking the test suite as failed because $TESTS_ERRORED tests errored!";
diff --git a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
index df1912d9fcd..9127f6676f6 100644
--- a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
+++ b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
@@ -44,6 +44,10 @@ public void testTokensPerSecond() {
             .forEach(model -> {
               LlamaModule mModule = new LlamaModule(model.getPath(), tokenizerPath, 0.8f);
 
+              // Just a hack to print the observed TPS for the mode
+              System.out.println("DEBUG: " + model.getName() + " " + model.getPath());
+              System.out.flush();
+
               int loadResult = mModule.load();
               // Check that the model can be load successfully
               assertEquals(0, loadResult);

From 6d4180914bd9c47bdc6516c69e206b72b69b655f Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Fri, 9 Aug 2024 21:50:02 -0700
Subject: [PATCH 04/17] Fix compilation error

---
 .../java/com/example/executorchllamademo/PerfTest.java       | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
index 9127f6676f6..3b18e7a7ce9 100644
--- a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
+++ b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
@@ -28,9 +28,6 @@ public class PerfTest implements LlamaCallback {
   private static final String RESOURCE_PATH = "/data/local/tmp/llama/";
   private static final String TOKENIZER_BIN = "tokenizer.bin";
 
-  // From https://github.com/pytorch/executorch/blob/main/examples/models/llama2/README.md
-  private static final Float EXPECTED_TPS = 10.0F;
-
   private final List<String> results = new ArrayList<>();
   private final List<Float> tokensPerSecond = new ArrayList<>();
 
@@ -39,7 +36,7 @@ public void testTokensPerSecond() {
     String tokenizerPath = RESOURCE_PATH + TOKENIZER_BIN;
     // Find out the model name
     File directory = new File(RESOURCE_PATH);
-    File[] files = Arrays.stream(directory.listFiles())
+    Arrays.stream(directory.listFiles())
             .filter(file -> file.getName().endsWith(".pte"))
             .forEach(model -> {
               LlamaModule mModule = new LlamaModule(model.getPath(), tokenizerPath, 0.8f);

From b92e471ea194ea97146d059e7be957e97512c85e Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Sat, 10 Aug 2024 00:03:48 -0700
Subject: [PATCH 05/17] Debug print

---
 .../android/LlamaDemo/app/build.gradle.kts    |  1 +
 .../example/executorchllamademo/PerfTest.java | 30 +++++++++----------
 2 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/examples/demo-apps/android/LlamaDemo/app/build.gradle.kts b/examples/demo-apps/android/LlamaDemo/app/build.gradle.kts
index 37c8cbf0ba2..c13dde2755e 100644
--- a/examples/demo-apps/android/LlamaDemo/app/build.gradle.kts
+++ b/examples/demo-apps/android/LlamaDemo/app/build.gradle.kts
@@ -65,6 +65,7 @@ dependencies {
   androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1")
   androidTestImplementation(platform("androidx.compose:compose-bom:2023.03.00"))
   androidTestImplementation("androidx.compose.ui:ui-test-junit4")
+  androidTestImplementation("androidx.test:monitor:1.7.1")
   debugImplementation("androidx.compose.ui:ui-tooling")
   debugImplementation("androidx.compose.ui:ui-test-manifest")
 }
diff --git a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
index 3b18e7a7ce9..57c57715a00 100644
--- a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
+++ b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
@@ -39,24 +39,19 @@ public void testTokensPerSecond() {
     Arrays.stream(directory.listFiles())
             .filter(file -> file.getName().endsWith(".pte"))
             .forEach(model -> {
-              LlamaModule mModule = new LlamaModule(model.getPath(), tokenizerPath, 0.8f);
+                LlamaModule mModule = new LlamaModule(model.getPath(), tokenizerPath, 0.8f);
 
-              // Just a hack to print the observed TPS for the mode
-              System.out.println("DEBUG: " + model.getName() + " " + model.getPath());
-              System.out.flush();
+                int loadResult = mModule.load();
+                // Check that the model can be load successfully
+                assertEquals(0, loadResult);
 
-              int loadResult = mModule.load();
-              // Check that the model can be load successfully
-              assertEquals(0, loadResult);
+                // Run a testing prompt
+                mModule.generate("How do you do! I'm testing llama2 on mobile device", PerfTest.this);
+                assertFalse(tokensPerSecond.isEmpty());
 
-              // Run a testing prompt
-              mModule.generate("How do you do! I'm testing llama2 on mobile device", PerfTest.this);
-              assertFalse(tokensPerSecond.isEmpty());
-
-              final Float tps = tokensPerSecond.get(tokensPerSecond.size() - 1);
-              // Just a hack to print the observed TPS for the model
-              System.out.println("The observed TPS for " + model.getName() + " is " + tps);
-              System.out.flush();
+                final Float tps = tokensPerSecond.get(tokensPerSecond.size() - 1);
+                // TODO: Figure out a way to print to instrument log output
+                assertTrue("The observed TPS for " + model.getName() + " is " + tps, false);
             });
   }
 
@@ -69,4 +64,9 @@ public void onResult(String result) {
   public void onStats(float tps) {
     tokensPerSecond.add(tps);
   }
+
+  // https://stackoverflow.com/questions/36425497/how-to-print-logs-in-cmd-console-while-execute-android-instrument-test
+  private void print(String msg) {
+      
+  }
 }

From 3f69c555bef48145ab44a38cd9046faddb166454 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Sat, 10 Aug 2024 02:09:06 -0700
Subject: [PATCH 06/17] Attempt to use sendStatus

---
 .../demo-apps/android/LlamaDemo/app/build.gradle.kts |  1 -
 .../com/example/executorchllamademo/PerfTest.java    | 12 +++++++-----
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/examples/demo-apps/android/LlamaDemo/app/build.gradle.kts b/examples/demo-apps/android/LlamaDemo/app/build.gradle.kts
index c13dde2755e..37c8cbf0ba2 100644
--- a/examples/demo-apps/android/LlamaDemo/app/build.gradle.kts
+++ b/examples/demo-apps/android/LlamaDemo/app/build.gradle.kts
@@ -65,7 +65,6 @@ dependencies {
   androidTestImplementation("androidx.test.espresso:espresso-core:3.5.1")
   androidTestImplementation(platform("androidx.compose:compose-bom:2023.03.00"))
   androidTestImplementation("androidx.compose.ui:ui-test-junit4")
-  androidTestImplementation("androidx.test:monitor:1.7.1")
   debugImplementation("androidx.compose.ui:ui-tooling")
   debugImplementation("androidx.compose.ui:ui-test-manifest")
 }
diff --git a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
index 57c57715a00..cf26894ad2b 100644
--- a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
+++ b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
@@ -13,6 +13,8 @@
 import static org.junit.Assert.assertFalse;
 
 import androidx.test.ext.junit.runners.AndroidJUnit4;
+import androidx.test.platform.app.InstrumentationRegistry;
+import android.os.Bundle;
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -50,8 +52,7 @@ public void testTokensPerSecond() {
                 assertFalse(tokensPerSecond.isEmpty());
 
                 final Float tps = tokensPerSecond.get(tokensPerSecond.size() - 1);
-                // TODO: Figure out a way to print to instrument log output
-                assertTrue("The observed TPS for " + model.getName() + " is " + tps, false);
+                reportTPS(tps);
             });
   }
 
@@ -65,8 +66,9 @@ public void onStats(float tps) {
     tokensPerSecond.add(tps);
   }
 
-  // https://stackoverflow.com/questions/36425497/how-to-print-logs-in-cmd-console-while-execute-android-instrument-test
-  private void print(String msg) {
-      
+  private void reportTPS(final Float tps) {
+    Bundle bundle = new Bundle();
+    bundle.putFloat("TPS", tps);
+    InstrumentationRegistry.getInstrumentation().sendStatus(0, bundle);
   }
 }

From 3c084ec13d977e50c782629206ea84df254a56db Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Sat, 10 Aug 2024 02:37:10 -0700
Subject: [PATCH 07/17] Report TPS via instrument status

---
 .../LlamaDemo/android-llama2-device-farm-test-spec.yml      | 2 +-
 .../java/com/example/executorchllamademo/PerfTest.java      | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
index 053ca171c78..1500473b6c8 100644
--- a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
+++ b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
@@ -50,7 +50,7 @@ phases:
           false;
         elif [ $TESTS_PASSED -ne 0 ];
         then
-          OBSERVED_TPS=$(grep "The observed TPS " $INSTRUMENT_LOG | tail -n 1)
+          OBSERVED_TPS=$(grep "TPS=" $INSTRUMENT_LOG | tail -n 1)
 
           if [ -n "${OBSERVED_TPS}" ];
           then
diff --git a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
index cf26894ad2b..a03cdc36c1c 100644
--- a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
+++ b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
@@ -52,7 +52,7 @@ public void testTokensPerSecond() {
                 assertFalse(tokensPerSecond.isEmpty());
 
                 final Float tps = tokensPerSecond.get(tokensPerSecond.size() - 1);
-                reportTPS(tps);
+                reportMetric("TPS", tps);
             });
   }
 
@@ -66,9 +66,9 @@ public void onStats(float tps) {
     tokensPerSecond.add(tps);
   }
 
-  private void reportTPS(final Float tps) {
+  private void reportMetric(final String metric, final Float value) {
     Bundle bundle = new Bundle();
-    bundle.putFloat("TPS", tps);
+    bundle.putFloat(metric, value);
     InstrumentationRegistry.getInstrumentation().sendStatus(0, bundle);
   }
 }

From 9372c8f8340e75cb1917b3d084e1887133bbdd8b Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Sat, 10 Aug 2024 02:45:07 -0700
Subject: [PATCH 08/17] Just use the spec from S3

---
 .github/workflows/android-perf.yml | 1 -
 .github/workflows/android.yml      | 1 -
 2 files changed, 2 deletions(-)

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index c93c3d1e848..d702065774e 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -218,7 +218,6 @@ jobs:
       # TODO: Hard code llm_demo_bpe for now in this job.
       android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_bpe/app-debug.apk
       android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_bpe/app-debug-androidTest.apk
-      # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
       test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
       # Uploaded to S3 from the previous job
       extra-data: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/${{ matrix.model }}_${{ matrix.delegate }}/model.zip
diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml
index 8d3440b7c15..f937040d922 100644
--- a/.github/workflows/android.yml
+++ b/.github/workflows/android.yml
@@ -170,7 +170,6 @@ jobs:
       # Uploaded to S3 from the previous job, the name of the app comes from the project itself
       android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug.apk
       android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug-androidTest.apk
-      # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
       test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
       # Among the input, this is the biggest file, so it is cached on AWS to make the test faster. Note that the file is deleted by AWS after 30
       # days and the job will automatically re-upload the file when that happens.

From 8a8aefe7d5480030a03d08f10be09ea7caca48c4 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Sat, 10 Aug 2024 02:57:18 -0700
Subject: [PATCH 09/17] It's working

---
 .github/workflows/android-perf.yml                             | 3 ++-
 .github/workflows/android.yml                                  | 3 ++-
 .../android/LlamaDemo/android-llama2-device-farm-test-spec.yml | 2 +-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index d702065774e..9e675e654ea 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -218,6 +218,7 @@ jobs:
       # TODO: Hard code llm_demo_bpe for now in this job.
       android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_bpe/app-debug.apk
       android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_bpe/app-debug-androidTest.apk
-      test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
+      # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
+      test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/3f4ca9bd-9523-4501-8649-fab506333675
       # Uploaded to S3 from the previous job
       extra-data: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/${{ matrix.model }}_${{ matrix.delegate }}/model.zip
diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml
index f937040d922..580bb5f63e0 100644
--- a/.github/workflows/android.yml
+++ b/.github/workflows/android.yml
@@ -170,7 +170,8 @@ jobs:
       # Uploaded to S3 from the previous job, the name of the app comes from the project itself
       android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug.apk
       android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug-androidTest.apk
-      test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
+      # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
+      test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/3f4ca9bd-9523-4501-8649-fab506333675
       # Among the input, this is the biggest file, so it is cached on AWS to make the test faster. Note that the file is deleted by AWS after 30
       # days and the job will automatically re-upload the file when that happens.
       extra-data: https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b-0717.zip
diff --git a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
index 1500473b6c8..2865cdf3088 100644
--- a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
+++ b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
@@ -50,7 +50,7 @@ phases:
           false;
         elif [ $TESTS_PASSED -ne 0 ];
         then
-          OBSERVED_TPS=$(grep "TPS=" $INSTRUMENT_LOG | tail -n 1)
+          OBSERVED_TPS=$(grep "INSTRUMENTATION_STATUS: TPS=" $INSTRUMENT_LOG | tail -n 1)
 
           if [ -n "${OBSERVED_TPS}" ];
           then

From 56508c4b50e19df77fac5c9c1bb46a282bf20284 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Sat, 10 Aug 2024 03:38:13 -0700
Subject: [PATCH 10/17] Also print the model name

---
 .../java/com/example/executorchllamademo/PerfTest.java | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
index a03cdc36c1c..5fb644b08fe 100644
--- a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
+++ b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
@@ -39,9 +39,11 @@ public void testTokensPerSecond() {
     // Find out the model name
     File directory = new File(RESOURCE_PATH);
     Arrays.stream(directory.listFiles())
-            .filter(file -> file.getName().endsWith(".pte"))
+            .filter(file -> file.getName().endsWith(".pte") || file.getName().endsWith(".pt"))
             .forEach(model -> {
                 LlamaModule mModule = new LlamaModule(model.getPath(), tokenizerPath, 0.8f);
+                // Print the model name because there might be more than one of them
+                report("ModelName", model.getName());
 
                 int loadResult = mModule.load();
                 // Check that the model can be load successfully
@@ -71,4 +73,10 @@ private void reportMetric(final String metric, final Float value) {
     bundle.putFloat(metric, value);
     InstrumentationRegistry.getInstrumentation().sendStatus(0, bundle);
   }
+
+  private void report(final String key, final String value) {
+    Bundle bundle = new Bundle();
+    bundle.putString(key, value);
+    InstrumentationRegistry.getInstrumentation().sendStatus(0, bundle);
+  }
 }

From 51dfaf37381d93fa2081333905427b1607eb41fa Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Sat, 10 Aug 2024 03:43:30 -0700
Subject: [PATCH 11/17] Fix lint

---
 .../example/executorchllamademo/PerfTest.java | 39 ++++++++++---------
 1 file changed, 21 insertions(+), 18 deletions(-)

diff --git a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
index 5fb644b08fe..cbd9db84a64 100644
--- a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
+++ b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
@@ -9,12 +9,14 @@
 package com.example.executorchllamademo;
 
 import static junit.framework.TestCase.assertTrue;
+
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 
+import android.os.Bundle;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import androidx.test.platform.app.InstrumentationRegistry;
-import android.os.Bundle;
+
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -39,23 +41,24 @@ public void testTokensPerSecond() {
     // Find out the model name
     File directory = new File(RESOURCE_PATH);
     Arrays.stream(directory.listFiles())
-            .filter(file -> file.getName().endsWith(".pte") || file.getName().endsWith(".pt"))
-            .forEach(model -> {
-                LlamaModule mModule = new LlamaModule(model.getPath(), tokenizerPath, 0.8f);
-                // Print the model name because there might be more than one of them
-                report("ModelName", model.getName());
-
-                int loadResult = mModule.load();
-                // Check that the model can be load successfully
-                assertEquals(0, loadResult);
-
-                // Run a testing prompt
-                mModule.generate("How do you do! I'm testing llama2 on mobile device", PerfTest.this);
-                assertFalse(tokensPerSecond.isEmpty());
-
-                final Float tps = tokensPerSecond.get(tokensPerSecond.size() - 1);
-                reportMetric("TPS", tps);
-            });
+        .filter(file -> file.getName().endsWith(".pte") || file.getName().endsWith(".pt"))
+        .forEach(
+          model -> {
+            LlamaModule mModule = new LlamaModule(model.getPath(), tokenizerPath, 0.8f);
+            // Print the model name because there might be more than one of them
+            report("ModelName", model.getName());
+
+            int loadResult = mModule.load();
+            // Check that the model can be load successfully
+            assertEquals(0, loadResult);
+
+            // Run a testing prompt
+            mModule.generate("How do you do! I'm testing llama2 on mobile device", PerfTest.this);
+            assertFalse(tokensPerSecond.isEmpty());
+
+            final Float tps = tokensPerSecond.get(tokensPerSecond.size() - 1);
+            reportMetric("TPS", tps);
+        });
   }
 
   @Override

From 2eff98c1c99f9e0e47a3d5d13dd5f91c2184983a Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Sat, 10 Aug 2024 03:48:55 -0700
Subject: [PATCH 12/17] Minor tweak

---
 .../java/com/example/executorchllamademo/PerfTest.java        | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
index cbd9db84a64..003f34729a8 100644
--- a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
+++ b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
@@ -57,7 +57,7 @@ public void testTokensPerSecond() {
             assertFalse(tokensPerSecond.isEmpty());
 
             final Float tps = tokensPerSecond.get(tokensPerSecond.size() - 1);
-            reportMetric("TPS", tps);
+            report("TPS", tps);
         });
   }
 
@@ -71,7 +71,7 @@ public void onStats(float tps) {
     tokensPerSecond.add(tps);
   }
 
-  private void reportMetric(final String metric, final Float value) {
+  private void report(final String metric, final Float value) {
     Bundle bundle = new Bundle();
     bundle.putFloat(metric, value);
     InstrumentationRegistry.getInstrumentation().sendStatus(0, bundle);

From 1c752fb6279ba860bcc72474474a0858e0ad3419 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Sat, 10 Aug 2024 07:25:54 -0700
Subject: [PATCH 13/17] Fix internal lint

---
 .../android-llama2-device-farm-test-spec.yml  |  2 ++
 .../example/executorchllamademo/PerfTest.java | 34 +++++++++----------
 2 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
index 2865cdf3088..e777ef38bef 100644
--- a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
+++ b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
@@ -13,8 +13,10 @@ phases:
       - adb -s $DEVICEFARM_DEVICE_UDID shell "mkdir -p /data/local/tmp/llama/"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.bin /data/local/tmp/llama/"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.pte /data/local/tmp/llama/"
+      - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.pt /data/local/tmp/llama/"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/*.bin"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/*.pte"
+      - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/*.pt"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /data/local/tmp/llama/"
 
   test:
diff --git a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
index 003f34729a8..881f6115e9b 100644
--- a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
+++ b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
@@ -8,7 +8,6 @@
 
 package com.example.executorchllamademo;
 
-import static junit.framework.TestCase.assertTrue;
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
@@ -16,7 +15,6 @@
 import android.os.Bundle;
 import androidx.test.ext.junit.runners.AndroidJUnit4;
 import androidx.test.platform.app.InstrumentationRegistry;
-
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -43,22 +41,22 @@ public void testTokensPerSecond() {
     Arrays.stream(directory.listFiles())
         .filter(file -> file.getName().endsWith(".pte") || file.getName().endsWith(".pt"))
         .forEach(
-          model -> {
-            LlamaModule mModule = new LlamaModule(model.getPath(), tokenizerPath, 0.8f);
-            // Print the model name because there might be more than one of them
-            report("ModelName", model.getName());
-
-            int loadResult = mModule.load();
-            // Check that the model can be load successfully
-            assertEquals(0, loadResult);
-
-            // Run a testing prompt
-            mModule.generate("How do you do! I'm testing llama2 on mobile device", PerfTest.this);
-            assertFalse(tokensPerSecond.isEmpty());
-
-            final Float tps = tokensPerSecond.get(tokensPerSecond.size() - 1);
-            report("TPS", tps);
-        });
+            model -> {
+              LlamaModule mModule = new LlamaModule(model.getPath(), tokenizerPath, 0.8f);
+              // Print the model name because there might be more than one of them
+              report("ModelName", model.getName());
+
+              int loadResult = mModule.load();
+              // Check that the model can be load successfully
+              assertEquals(0, loadResult);
+
+              // Run a testing prompt
+              mModule.generate("How do you do! I'm testing llama2 on mobile device", PerfTest.this);
+              assertFalse(tokensPerSecond.isEmpty());
+
+              final Float tps = tokensPerSecond.get(tokensPerSecond.size() - 1);
+              report("TPS", tps);
+            });
   }
 
   @Override

From 47e413b9906ff2869327301d7b4470d48104bb37 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Sat, 10 Aug 2024 07:35:14 -0700
Subject: [PATCH 14/17] Update spec one more time

---
 .github/workflows/android-perf.yml | 2 +-
 .github/workflows/android.yml      | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index 9e675e654ea..c93c3d1e848 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -219,6 +219,6 @@ jobs:
       android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_bpe/app-debug.apk
       android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_bpe/app-debug-androidTest.apk
       # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
-      test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/3f4ca9bd-9523-4501-8649-fab506333675
+      test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
       # Uploaded to S3 from the previous job
       extra-data: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/${{ matrix.model }}_${{ matrix.delegate }}/model.zip
diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml
index 580bb5f63e0..8d3440b7c15 100644
--- a/.github/workflows/android.yml
+++ b/.github/workflows/android.yml
@@ -171,7 +171,7 @@ jobs:
       android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug.apk
       android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug-androidTest.apk
       # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
-      test-spec: arn:aws:devicefarm:us-west-2:308535385114:upload:02a2cf0f-6d9b-45ee-ba1a-a086587469e6/3f4ca9bd-9523-4501-8649-fab506333675
+      test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
       # Among the input, this is the biggest file, so it is cached on AWS to make the test faster. Note that the file is deleted by AWS after 30
       # days and the job will automatically re-upload the file when that happens.
       extra-data: https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b-0717.zip

From c8b5c94cb461830cfb9e663e93d8a860a5a331f6 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Sat, 10 Aug 2024 11:26:34 -0700
Subject: [PATCH 15/17] Check for passing test last

---
 .../android-llama2-device-farm-test-spec.yml  | 21 ++++++++++---------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
index e777ef38bef..2060144f257 100644
--- a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
+++ b/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
@@ -50,16 +50,6 @@ phases:
         then
           echo "[PyTorch] Marking the test suite as failed because no tests started!";
           false;
-        elif [ $TESTS_PASSED -ne 0 ];
-        then
-          OBSERVED_TPS=$(grep "INSTRUMENTATION_STATUS: TPS=" $INSTRUMENT_LOG | tail -n 1)
-
-          if [ -n "${OBSERVED_TPS}" ];
-          then
-            echo "[PyTorch] ${OBSERVED_TPS}";
-          else
-            echo "[PyTorch] Test passes but couldn't find the observed TPS from instrument log";
-          fi
         elif [ $TESTS_FAILED -ne 0 ];
         then
           echo "[PyTorch] Marking the test suite as failed because it failed to load the model";
@@ -72,6 +62,17 @@ phases:
         then
           echo "[PyTorch] Marking the test suite as failed because the app crashed due to OOM!";
           false;
+        # Check for this last to make sure that there is no failure
+        elif [ $TESTS_PASSED -ne 0 ];
+        then
+          OBSERVED_TPS=$(grep "INSTRUMENTATION_STATUS: TPS=" $INSTRUMENT_LOG | tail -n 1)
+
+          if [ -n "${OBSERVED_TPS}" ];
+          then
+            echo "[PyTorch] ${OBSERVED_TPS}";
+          else
+            echo "[PyTorch] Test passes but couldn't find the observed TPS from instrument log";
+          fi
         fi;
 
   post_test:

From 2d3c2c08ba10178f42cf449b1530e284df3c6deb Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Sat, 10 Aug 2024 11:28:42 -0700
Subject: [PATCH 16/17] Lint yet

---
 .../java/com/example/executorchllamademo/PerfTest.java           | 1 -
 1 file changed, 1 deletion(-)

diff --git a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
index 881f6115e9b..6bec9e97bf2 100644
--- a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
+++ b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
@@ -8,7 +8,6 @@
 
 package com.example.executorchllamademo;
 
-
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 

From d08a5ebb59552a63c151e215d3d5f6de91b5e181 Mon Sep 17 00:00:00 2001
From: Huy Do <huydhn@gmail.com>
Date: Mon, 12 Aug 2024 10:56:38 -0700
Subject: [PATCH 17/17] Address review comments

---
 .github/workflows/android-perf.yml                             | 3 +--
 .github/workflows/android.yml                                  | 3 +--
 ...arm-test-spec.yml => android-llm-device-farm-test-spec.yml} | 2 --
 .../java/com/example/executorchllamademo/PerfTest.java         | 2 +-
 4 files changed, 3 insertions(+), 7 deletions(-)
 rename examples/demo-apps/android/LlamaDemo/{android-llama2-device-farm-test-spec.yml => android-llm-device-farm-test-spec.yml} (95%)

diff --git a/.github/workflows/android-perf.yml b/.github/workflows/android-perf.yml
index c93c3d1e848..4f8b216a545 100644
--- a/.github/workflows/android-perf.yml
+++ b/.github/workflows/android-perf.yml
@@ -218,7 +218,6 @@ jobs:
       # TODO: Hard code llm_demo_bpe for now in this job.
       android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_bpe/app-debug.apk
       android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_bpe/app-debug-androidTest.apk
-      # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
-      test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
+      test-spec: https://ossci-assets.s3.amazonaws.com/android-llm-device-farm-test-spec.yml
       # Uploaded to S3 from the previous job
       extra-data: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/${{ matrix.model }}_${{ matrix.delegate }}/model.zip
diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml
index 8d3440b7c15..5af09dc490a 100644
--- a/.github/workflows/android.yml
+++ b/.github/workflows/android.yml
@@ -170,8 +170,7 @@ jobs:
       # Uploaded to S3 from the previous job, the name of the app comes from the project itself
       android-app-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug.apk
       android-test-archive: https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifact/llm_demo_${{ matrix.tokenizer }}/app-debug-androidTest.apk
-      # The test spec can be downloaded from https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
-      test-spec: https://ossci-assets.s3.amazonaws.com/android-llama2-device-farm-test-spec-v2.yml
+      test-spec: https://ossci-assets.s3.amazonaws.com/android-llm-device-farm-test-spec.yml
       # Among the input, this is the biggest file, so it is cached on AWS to make the test faster. Note that the file is deleted by AWS after 30
       # days and the job will automatically re-upload the file when that happens.
       extra-data: https://ossci-assets.s3.amazonaws.com/executorch-android-llama2-7b-0717.zip
diff --git a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml b/examples/demo-apps/android/LlamaDemo/android-llm-device-farm-test-spec.yml
similarity index 95%
rename from examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
rename to examples/demo-apps/android/LlamaDemo/android-llm-device-farm-test-spec.yml
index 2060144f257..cac83b8e6f5 100644
--- a/examples/demo-apps/android/LlamaDemo/android-llama2-device-farm-test-spec.yml
+++ b/examples/demo-apps/android/LlamaDemo/android-llm-device-farm-test-spec.yml
@@ -13,10 +13,8 @@ phases:
       - adb -s $DEVICEFARM_DEVICE_UDID shell "mkdir -p /data/local/tmp/llama/"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.bin /data/local/tmp/llama/"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.pte /data/local/tmp/llama/"
-      - adb -s $DEVICEFARM_DEVICE_UDID shell "mv /sdcard/*.pt /data/local/tmp/llama/"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/*.bin"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/*.pte"
-      - adb -s $DEVICEFARM_DEVICE_UDID shell "chmod 664 /data/local/tmp/llama/*.pt"
       - adb -s $DEVICEFARM_DEVICE_UDID shell "ls -la /data/local/tmp/llama/"
 
   test:
diff --git a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
index 6bec9e97bf2..221a9bd7417 100644
--- a/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
+++ b/examples/demo-apps/android/LlamaDemo/app/src/androidTest/java/com/example/executorchllamademo/PerfTest.java
@@ -38,7 +38,7 @@ public void testTokensPerSecond() {
     // Find out the model name
     File directory = new File(RESOURCE_PATH);
     Arrays.stream(directory.listFiles())
-        .filter(file -> file.getName().endsWith(".pte") || file.getName().endsWith(".pt"))
+        .filter(file -> file.getName().endsWith(".pte"))
         .forEach(
             model -> {
               LlamaModule mModule = new LlamaModule(model.getPath(), tokenizerPath, 0.8f);