diff --git a/ASLRecognition/README.md b/ASLRecognition/README.md index f118a436..da291363 100644 --- a/ASLRecognition/README.md +++ b/ASLRecognition/README.md @@ -6,9 +6,9 @@ ## Prerequisites -* PyTorch 1.9.0 and torchvision 0.10.0 (Optional) +* PyTorch 1.10.0 and torchvision 0.11.1 (Optional) * Python 3.8 or above (Optional) -* Android Pytorch library pytorch_android_lite:1.9.0, pytorch_android_torchvision:1.9.0 +* Android Pytorch library pytorch_android_lite:1.10.0, pytorch_android_torchvision_lite:1.10.0 * Android Studio 4.0.1 or later ## Quick Start @@ -17,9 +17,9 @@ To Test Run the ASL recognition Android App, follow the steps below: ### 1. Train and Prepare the Model -If you don't have the PyTorch 1.9.0 and torchvision 0.10.0 installed, or if don't want to install them, you can skip this step. The trained, scripted and optimized model is already included in the repo, located at `ASLRecognitionapp/src/main/assets`. +If you don't have the PyTorch 1.10.0 and torchvision 0.11.1 installed, or if don't want to install them, you can skip this step. The trained, scripted and optimized model is already included in the repo, located at `ASLRecognitionapp/src/main/assets`. -Otherwise, open a terminal window, make sure you have torch 1.9.0 and torchvision 0.10.0 installed using command like `pip list|grep torch`, or install them using command like `pip install torch torchvision`, then run the following commands: +Otherwise, open a terminal window, make sure you have torch 1.10.0 and torchvision 0.11.1 installed using command like `pip list|grep torch`, or install them using command like `pip install torch torchvision`, then run the following commands: ``` git clone https://github.com/pytorch/android-demo-app @@ -29,6 +29,7 @@ cd android-demo-app/ASLRecognition/scripts Download the ASL alphabet dataset [here](https://www.kaggle.com/grassknoted/asl-alphabet) and unzip it into the `ASLRecognition/scripts` folder. Then run the scripts below, which are based on this [tutorial](https://debuggercafe.com/american-sign-language-detection-using-deep-learning/), to pre-process the training images, train the model and convert and optimize the trained model to the mobile interpreter model: ``` +pip install opencv-python pandas sklearn imutils matplotlib python preprocess_image.py python create_csv.py python train.py --epochs 5 # on a machine without GPU this can take hours @@ -51,8 +52,8 @@ For more information on how to use a test script like the above to find out the Open the ASLRecognition project using Android Studio. Note the app's `build.gradle` file has the following lines: ``` -implementation 'org.pytorch:pytorch_android_lite:1.9.0' -implementation 'org.pytorch:pytorch_android_torchvision:1.9.0' +implementation 'org.pytorch:pytorch_android_lite:1.10.0' +implementation 'org.pytorch:pytorch_android_torchvision_lite:1.10.0' ``` and in the MainActivity.java, the code below is used to load the model: @@ -68,7 +69,7 @@ Select an Android emulator or device and build and run the app. Some of the 26 t ![](screenshot2.png) ![](screenshot2.png) -To test the live ASL alphabet gesture recognition, after you get familiar with the 26 ASL signs by tapping Next and Recognize, select the LIVE button and make some ASL gesture in front of the camera. A screencast of the app running is available [here](https://drive.google.com/file/d/1NxehGHlU-RiYP_JU9qkpCEcQR2hG-vyv/view?usp=sharing). +To test the live ASL alphabet gesture recognition, after you get familiar with the 26 ASL signs by tapping Next and Recognize, select the LIVE button and make some ASL gesture in front of the camera. A screencast of the app running is available [here](https://drive.google.com/file/d/1NxehGHlU-RiYP_JU9qkpCEcQR2hG-vyv/view?usp=sharing). ### 4. What's Next With a different sign language dataset such as the RWTH-PHOENIX-Weather 2014 MS [Public Hand Shape Dataset](https://www-i6.informatik.rwth-aachen.de/~koller/1miohands-data/) or the [Continuous Sign Language Recognition Dataset](https://www-i6.informatik.rwth-aachen.de/~koller/RWTH-PHOENIX/) and a state-of-the-art [sign language transformer](https://arxiv.org/pdf/2003.13830v1.pdf) based model, more powerful sign language recognition Android app can be developed based on the app here. diff --git a/ASLRecognition/app/build.gradle b/ASLRecognition/app/build.gradle index f64b72f6..1276baca 100644 --- a/ASLRecognition/app/build.gradle +++ b/ASLRecognition/app/build.gradle @@ -40,6 +40,6 @@ dependencies { implementation "androidx.camera:camera-core:$camerax_version" implementation "androidx.camera:camera-camera2:$camerax_version" - implementation 'org.pytorch:pytorch_android_lite:1.9.0' - implementation 'org.pytorch:pytorch_android_torchvision:1.9.0' + implementation 'org.pytorch:pytorch_android_lite:1.10.0' + implementation 'org.pytorch:pytorch_android_torchvision_lite:1.10.0' } \ No newline at end of file diff --git a/ASLRecognition/app/src/main/assets/asl.ptl b/ASLRecognition/app/src/main/assets/asl.ptl index f896ec19..b4412261 100644 Binary files a/ASLRecognition/app/src/main/assets/asl.ptl and b/ASLRecognition/app/src/main/assets/asl.ptl differ diff --git a/ASLRecognition/scripts/test.py b/ASLRecognition/scripts/test.py index af685233..038da9ac 100644 --- a/ASLRecognition/scripts/test.py +++ b/ASLRecognition/scripts/test.py @@ -8,18 +8,20 @@ import numpy as np import cv2 import argparse -import albumentations +import torchvision.transforms as transforms import torch.nn.functional as F import time import cnn_models +from PIL import Image + # construct the argument parser and parse the arguments parser = argparse.ArgumentParser() parser.add_argument('-i', '--img', default='../app/src/main/assets/C1.jpg', type=str, help='path for the image to test on') args = vars(parser.parse_args()) -aug = albumentations.Compose([ - albumentations.Resize(224, 224, always_apply=True), +aug = transforms.Compose([ + transforms.Resize((224, 224)), ]) # load label binarizer @@ -29,10 +31,8 @@ print(model) print('Model loaded') -image = cv2.imread(f"{args['img']}") -image_copy = image.copy() - -image = aug(image=np.array(image))['image'] +image = Image.open(f"{args['img']}") +image = aug(image) image = np.transpose(image, (2, 0, 1)).astype(np.float32) image = torch.tensor(image, dtype=torch.float) image = image.unsqueeze(0) diff --git a/D2Go/ObjectDetection/app/build.gradle b/D2Go/ObjectDetection/app/build.gradle index ec9ad301..c8d5756c 100644 --- a/D2Go/ObjectDetection/app/build.gradle +++ b/D2Go/ObjectDetection/app/build.gradle @@ -45,7 +45,7 @@ android { packagingOptions { - doNotStrip '**.so' + pickFirst "**" } } @@ -64,7 +64,7 @@ dependencies { implementation "androidx.camera:camera-core:$camerax_version" implementation "androidx.camera:camera-camera2:$camerax_version" - implementation 'org.pytorch:pytorch_android:1.8.0' - implementation 'org.pytorch:pytorch_android_torchvision:1.8.0' - implementation 'org.pytorch:torchvision_ops:0.9.0' + implementation 'org.pytorch:pytorch_android_lite:1.10.0' + implementation 'org.pytorch:pytorch_android_torchvision_lite:1.10.0' + implementation 'org.pytorch:torchvision_ops:0.10.0' } \ No newline at end of file diff --git a/D2Go/README.md b/D2Go/README.md index 5ff891e3..f00908d4 100644 --- a/D2Go/README.md +++ b/D2Go/README.md @@ -8,16 +8,16 @@ This D2Go Android demo app shows how to prepare and use the D2Go model on Androi ## Prerequisites -* PyTorch 1.8.0 and torchvision 0.9.0 (Optional) +* PyTorch 1.10.0 and torchvision 0.11.1 (Optional) * Python 3.8 or above (Optional) -* Android Pytorch library 1.8.0, torchvision library 1.8.0, torchvision_ops library 0.9.0 +* Android Pytorch library pytorch_android_lite 1.10.0, pytorch_android_torchvision_lite 1.10.0, torchvision_ops library 0.10.0 * Android Studio 4.0.1 or later ## Quick Start This section shows how to create and use the D2Go model and the pre-built torchvision-ops library in a completed Android app. To just build and run the app without creating the D2Go model yourself, go directly to Step 4. -1. Install PyTorch 1.8.0 and torchvision 0.9.0, for example: +1. Install PyTorch 1.10.0 and torchvision 0.11.1, for example: ``` conda create -n d2go python=3.8.5 @@ -54,9 +54,9 @@ In Android Studio, open `android-demo-app/D2Go` (not `android-demo-app/D2Go/Obje The main changes needed to use the D2Go model and the required and pre-built torchvision-ops library are adding ``` -implementation 'org.pytorch:pytorch_android:1.8.0' -implementation 'org.pytorch:pytorch_android_torchvision:1.8.0' -implementation 'org.pytorch:torchvision_ops:0.9.0' +implementation 'org.pytorch:pytorch_android_lite:1.10.0' +implementation 'org.pytorch:pytorch_android_torchvision_lite:1.10.0' +implementation 'org.pytorch:torchvision_ops:0.10.0' ``` in the build.gradle file and ``` diff --git a/D2Go/build.gradle b/D2Go/build.gradle index 2457d277..5fde2be0 100644 --- a/D2Go/build.gradle +++ b/D2Go/build.gradle @@ -24,7 +24,7 @@ allprojects { } dependencies { - classpath 'com.android.tools.build:gradle:3.3.2' + classpath 'com.android.tools.build:gradle:4.0.1' } } diff --git a/ImageSegmentation/README.md b/ImageSegmentation/README.md index 66efe863..517f73e8 100644 --- a/ImageSegmentation/README.md +++ b/ImageSegmentation/README.md @@ -6,9 +6,9 @@ This repo offers a Python script that converts the [PyTorch DeepLabV3 model](htt ## Prerequisites -* PyTorch 1.9.0 and torchvision 0.10.0 (Optional) +* PyTorch 1.10.0 and torchvision 0.11.1 (Optional) * Python 3.8 or above (Optional) -* Android Pytorch library pytorch_android_lite:1.9.0, pytorch_android_torchvision:1.9.0 +* Android Pytorch library pytorch_android_lite:1.10.0, pytorch_android_torchvision_lite:1.10.0 * Android Studio 4.0.1 or later ## Quick Start @@ -17,9 +17,9 @@ To Test Run the Image Segmentation Android App, follow the steps below: ### 1. Prepare the Model -If you don't have the PyTorch 1.9.0 environment set up, you can download the optimized-for-mobile Mobile Interpreter version of model file to the `android-demo-app/ImageSegmentation/app/src/main/assets` folder using the link [here](https://drive.google.com/file/d/1FCm-pHsLiPiiXBsJwookAa0VFS2zTgv-/view?usp=sharing). +If you don't have the PyTorch 1.10.0 environment set up, you can download the optimized-for-mobile Mobile Interpreter version of model file to the `android-demo-app/ImageSegmentation/app/src/main/assets` folder using the link [here](https://pytorch-mobile-demo-apps.s3.us-east-2.amazonaws.com/deeplabv3_scripted.pt). -Otherwise, open a terminal window, first install PyTorch 1.9.0 and torchvision 0.10.0 using command like `pip install torch torchvision`, then run the following commands: +Otherwise, open a terminal window, first install PyTorch 1.10.0 and torchvision 0.11.1 using command like `pip install torch torchvision`, then run the following commands: ``` git clone https://github.com/pytorch/android-demo-app @@ -34,8 +34,8 @@ The Python script `deeplabv3.py` is used to generate the TorchScript-formatted m Open the ImageSegmentation project using Android Studio. Note the app's `build.gradle` file has the following lines: ``` -implementation 'org.pytorch:pytorch_android_lite:1.9.0' -implementation 'org.pytorch:pytorch_android_torchvision:1.9.0' +implementation 'org.pytorch:pytorch_android_lite:1.10.0' +implementation 'org.pytorch:pytorch_android_torchvision_lite:1.10.0' ``` and in the MainActivity.java, the code below is used to load the model: diff --git a/ImageSegmentation/app/build.gradle b/ImageSegmentation/app/build.gradle index 88912581..8a38847f 100644 --- a/ImageSegmentation/app/build.gradle +++ b/ImageSegmentation/app/build.gradle @@ -35,6 +35,6 @@ dependencies { androidTestImplementation 'androidx.test.ext:junit:1.1.2' androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' - implementation 'org.pytorch:pytorch_android_lite:1.9.0' - implementation 'org.pytorch:pytorch_android_torchvision:1.9.0' + implementation 'org.pytorch:pytorch_android_lite:1.10.0' + implementation 'org.pytorch:pytorch_android_torchvision_lite:1.10.0' } \ No newline at end of file diff --git a/ImageSegmentation/deeplabv3.py b/ImageSegmentation/deeplabv3.py index 6c8f3876..d27d2893 100644 --- a/ImageSegmentation/deeplabv3.py +++ b/ImageSegmentation/deeplabv3.py @@ -1,7 +1,7 @@ import torch from torch.utils.mobile_optimizer import optimize_for_mobile -model = torch.hub.load('pytorch/vision:v0.9.0', 'deeplabv3_resnet50', pretrained=True) +model = torch.hub.load('pytorch/vision:v0.11.0', 'deeplabv3_resnet50', pretrained=True) model.eval() scripted_module = torch.jit.script(model) diff --git a/ObjectDetection/README.md b/ObjectDetection/README.md index 4d69a052..c17be305 100644 --- a/ObjectDetection/README.md +++ b/ObjectDetection/README.md @@ -8,9 +8,9 @@ ## Prerequisites -* PyTorch 1.9.0 or later (Optional) +* PyTorch 1.10.0 and torchvision 0.11.1 (Optional) * Python 3.8 (Optional) -* Android Pytorch library pytorch_android_lite:1.9.0 and pytorch_android_torchvision:1.9.0 +* Android Pytorch library pytorch_android_lite:1.10.0, pytorch_android_torchvision_lite:1.10.0 * Android Studio 4.0.1 or later ## Quick Start @@ -19,9 +19,7 @@ To Test Run the Object Detection Android App, follow the steps below: ### 1. Prepare the model -If you don't have the PyTorch environment set up to run the script, you can download the model file `yolov5s.torchscript.ptl` [here](https://drive.google.com/u/1/uc?id=1_MF7NVi9Csm1lizoSCp1wCtUUUpuhwet&export=download) to the `android-demo-app/ObjectDetection/app/src/main/assets` folder, then skip the rest of this step and go to step 2 directly. - -Be aware that the downloadable model file was created with PyTorch 1.9.0, matching the PyTorch Android library 1.9.0 specified in the project's `build.gradle` file as `implementation 'org.pytorch:pytorch_android_lite:1.9.0'`. If you use a different version of PyTorch to create your model by following the instructions below, make sure you specify the same PyTorch Android library version in the `build.gradle` file to avoid possible errors caused by the version mismatch. Furthermore, if you want to use the latest PyTorch master code to create the model, follow the steps at [Building PyTorch Android from Source](https://pytorch.org/mobile/android/#building-pytorch-android-from-source) and [Using the PyTorch Android Libraries Built](https://pytorch.org/mobile/android/#using-the-pytorch-android-libraries-built-from-source-or-nightly) on how to use the model in Android. +If you don't have the PyTorch environment set up to run the script, you can download the model file `yolov5s.torchscript.ptl` [here](https://pytorch-mobile-demo-apps.s3.us-east-2.amazonaws.com/yolov5s.torchscript.ptl) to the `android-demo-app/ObjectDetection/app/src/main/assets` folder, then skip the rest of this step and go to step 2 directly. The Python script `export.py` in the `models` folder of the [YOLOv5 repo](https://github.com/ultralytics/yolov5) is used to generate a TorchScript-formatted YOLOv5 model named `yolov5s.torchscript.pt` for mobile apps. @@ -51,7 +49,12 @@ Note that small sized version of the YOLOv5 model, which runs faster but with le ### 2. Build with Android Studio -Start Android Studio, then open the project located in `android-demo-app/ObjectDetection` +Start Android Studio, then open the project located in `android-demo-app/ObjectDetection`. Note the app's `build.gradle` file has the following lines: + +``` +implementation 'org.pytorch:pytorch_android_lite:1.10.0' +implementation 'org.pytorch:pytorch_android_torchvision_lite:1.10.0' +``` ### 3. Run the app diff --git a/ObjectDetection/app/build.gradle b/ObjectDetection/app/build.gradle index 8a8cd81d..80ed949f 100644 --- a/ObjectDetection/app/build.gradle +++ b/ObjectDetection/app/build.gradle @@ -37,6 +37,6 @@ dependencies { implementation "androidx.camera:camera-core:$camerax_version" implementation "androidx.camera:camera-camera2:$camerax_version" - implementation 'org.pytorch:pytorch_android_lite:1.9.0' - implementation 'org.pytorch:pytorch_android_torchvision:1.9.0' + implementation 'org.pytorch:pytorch_android_lite:1.10.0' + implementation 'org.pytorch:pytorch_android_torchvision_lite:1.10.0' } \ No newline at end of file diff --git a/QuestionAnswering/README.md b/QuestionAnswering/README.md index de88bb01..8b503615 100644 --- a/QuestionAnswering/README.md +++ b/QuestionAnswering/README.md @@ -8,28 +8,25 @@ In this demo app, written in Kotlin, we'll show how to quantize and convert the ## Prerequisites -* PyTorch 1.9.0 or later (Optional) +* PyTorch 1.10.0 or later (Optional) * Python 3.8 (Optional) -* Android Pytorch library org.pytorch:pytorch_android_lite:1.9.0 +* Android Pytorch library org.pytorch:pytorch_android_lite:1.10.0 * Android Studio 4.0.1 or later ## Quick Start -To Test Run the Android QA App, run the following commands on a Terminal: +To Test Run the Android QA App, follow the steps below: ### 1. Prepare the Model -If you don't have PyTorch installed or want to have a quick try of the demo app, you can download the scripted QA model `qa360_quantized.ptl` [here](https://drive.google.com/file/d/1PgD3pAEf0riUiT3BfwHOm6UEGk8FfJzI/view?usp=sharing) and save it to the `QuestionAnswering/app/src/main/assets` folder, then continue to Step 2. +If you don't have PyTorch installed or want to have a quick try of the demo app, you can download the scripted QA model `qa360_quantized.ptl` [here](https://pytorch-mobile-demo-apps.s3.us-east-2.amazonaws.com/qa360_quantized.ptl) and save it to the `QuestionAnswering/app/src/main/assets` folder, then continue to Step 2. -Be aware that the downloadable model file was created with PyTorch 1.9.0, matching the PyTorch Android library 1.9.0 specified in the project's `build.gradle` file as `implementation 'org.pytorch:pytorch_android:1.9.0'`. If you use a different version of PyTorch to create your model by following the instructions below, make sure you specify the same PyTorch Android library version in the `build.gradle` file to avoid possible errors caused by the version mismatch. Furthermore, if you want to use the latest PyTorch master code to create the model, follow the steps at [Building PyTorch Android from Source](https://pytorch.org/mobile/android/#building-pytorch-android-from-source) and [Using the PyTorch Android Libraries Built](https://pytorch.org/mobile/android/#using-the-pytorch-android-libraries-built-from-source-or-nightly) on how to use the model in Android. - -With PyTorch 1.9.0 installed, first install the Huggingface `transformers` by running `pip install transformers`, then run `python convert_distilbert_qa.py`. +With PyTorch 1.10.0 installed, first install the Huggingface `transformers` by running `pip install transformers`, then run `python convert_distilbert_qa.py`. Note that a pre-defined question and text, resulting in the size of the input tokens (of question and text) being 360, is used in the `convert_distilbert_qa.py`, and 360 is the maximum token size for the user text and question in the app. If the token size of the inputs of the text and question is less than 360, padding will be needed to make the model work correctly. After the script completes, copy the model file `qa360_quantized.ptl` to the Android app's assets folder. - ### 2. Build and run with Android Studio Start Android Studio, open the project located in `android-demo-app/QuestionAnswering`, and run on your AVD or real Android device. See this [video](https://drive.google.com/file/d/10hwGNFo5tylalKwut_CWFPJmV7JRdDKF/view?usp=sharing) for a screencast of the app running. Some example translation results are: diff --git a/QuestionAnswering/app/build.gradle b/QuestionAnswering/app/build.gradle index 4c2a2fcf..6c889bfe 100644 --- a/QuestionAnswering/app/build.gradle +++ b/QuestionAnswering/app/build.gradle @@ -32,11 +32,10 @@ dependencies { implementation 'androidx.appcompat:appcompat:1.2.0' implementation 'androidx.constraintlayout:constraintlayout:2.0.4' - implementation 'org.pytorch:pytorch_android_lite:1.9.0' - implementation "androidx.core:core-ktx:+" + implementation "androidx.core:core-ktx:1.6.0" implementation "org.jetbrains.kotlin:kotlin-stdlib-jdk7:$kotlin_version" - + implementation 'org.pytorch:pytorch_android_lite:1.10.0' } repositories { mavenCentral() diff --git a/Seq2SeqNMT/README.md b/Seq2SeqNMT/README.md index e697a35d..7ec08d29 100644 --- a/Seq2SeqNMT/README.md +++ b/Seq2SeqNMT/README.md @@ -12,9 +12,9 @@ This Android demo app shows: ## Prerequisites -* PyTorch 1.9.0 (Optional) +* PyTorch 1.10.0 (Optional) * Python 3.8 (Optional) -* Android Pytorch library org.pytorch:pytorch_android_lite:1.9.0 +* Android Pytorch library org.pytorch:pytorch_android_lite:1.10.0 * Android Studio 4.0.1 or later ## Quick Start @@ -25,7 +25,7 @@ To Test Run the Object Detection Android App, follow the steps below: If you don't have the PyTorch environment set up to run the script, you can download the PyTorch trained and optimized NMT encoder and decoder models compressed in a zip [here](https://drive.google.com/file/d/1S75cWNEp43U6nCp2MOBR-jE-ZnlHz1PI/view?usp=sharing), unzip it, copy them to the Android app's assets folder, and skip the rest of this step and go to step 2 directly. -If you have a good GPU and want to train your model from scratch, run `python seq2seq2_nmt.py` to go through the whole process of training, saving, loading, optimizing and saving the final mobile-ready models `optimized_encoder_150k.ptl` and `optimized_decoder_150k.ptl`. Copy the two model files to the Android app's assets folder. +If you have a good GPU and want to train your model from scratch, run `python seq2seq_nmt.py` to go through the whole process of training, saving, loading, optimizing and saving the final mobile-ready models `optimized_encoder_150k.ptl` and `optimized_decoder_150k.ptl`. Copy the two model files to the Android app's assets folder. ### 2. Build and run with Android Studio diff --git a/Seq2SeqNMT/app/build.gradle b/Seq2SeqNMT/app/build.gradle index 417523ae..45d860c9 100644 --- a/Seq2SeqNMT/app/build.gradle +++ b/Seq2SeqNMT/app/build.gradle @@ -33,6 +33,6 @@ dependencies { androidTestImplementation 'androidx.test.ext:junit:1.1.1' androidTestImplementation 'androidx.test.espresso:espresso-core:3.2.0' - implementation 'org.pytorch:pytorch_android_lite:1.9.0' + implementation 'org.pytorch:pytorch_android_lite:1.10.0' } \ No newline at end of file diff --git a/SpeechRecognition/README.md b/SpeechRecognition/README.md index 2749f36b..f7e36451 100644 --- a/SpeechRecognition/README.md +++ b/SpeechRecognition/README.md @@ -8,9 +8,9 @@ In this demo app, we'll show how to quantize, trace, and optimize the wav2vec2 m ## Prerequisites -* PyTorch 1.9.0 and torchaudio 0.9.0 (Optional) +* PyTorch 1.10.0 and torchaudio 0.10.0 (Optional) * Python 3.8 (Optional) -* Android Pytorch library org.pytorch:pytorch_android_lite:1.9.0 +* Android Pytorch library org.pytorch:pytorch_android_lite:1.10.0 * Android Studio 4.0.1 or later ## Quick Start @@ -24,20 +24,19 @@ git clone https://github.com/pytorch/android-demo-app cd android-demo-app/SpeechRecognition ``` -If you don't have PyTorch 1.9.0 and torchaudio 0.9.0 installed or want to have a quick try of the demo app, you can download the quantized scripted wav2vec2 model file [here](https://drive.google.com/file/d/1xMh-BZMSIeoohBfZvQFYcemmh5zUn_gh/view?usp=sharing), then drag and drop it to the `app/src/main/assets` folder inside `android-demo-app/SpeechRecognition`, and continue to Step 3. +If you don't have PyTorch 1.10.0 and torchaudio 0.10.0 installed or want to have a quick try of the demo app, you can download the quantized scripted wav2vec2 model file [here](https://drive.google.com/file/d/1xMh-BZMSIeoohBfZvQFYcemmh5zUn_gh/view?usp=sharing), then drag and drop it to the `app/src/main/assets` folder inside `android-demo-app/SpeechRecognition`, and continue to Step 3. ### 2. Prepare the Model -To install PyTorch 1.9.0, torchaudio 0.9.0 and the Hugging Face transformers, you can do something like this: +To install PyTorch 1.10.0, torchaudio 0.10.0 and the Hugging Face transformers, you can do something like this: ``` conda create -n wav2vec2 python=3.8.5 conda activate wav2vec2 -pip install torch torchaudio -pip install transformers +pip install torch torchaudio transformers ``` -Now with PyTorch 1.9.0 and torchaudio 0.9.0 installed, run the following commands on a Terminal: +Now with PyTorch 1.10.0 and torchaudio 0.10.0 installed, run the following commands on a Terminal: ``` python create_wav2vec2.py @@ -46,7 +45,7 @@ This will create the PyTorch mobile interpreter model file `wav2vec2.ptl`. Copy ``` mkdir -p app/src/main/assets -cp wav2vec2.pt app/src/main/assets +cp wav2vec2.ptl app/src/main/assets ``` ### 2. Build and run with Android Studio diff --git a/SpeechRecognition/app/build.gradle b/SpeechRecognition/app/build.gradle index 101aa0bd..35a7706a 100644 --- a/SpeechRecognition/app/build.gradle +++ b/SpeechRecognition/app/build.gradle @@ -40,6 +40,5 @@ dependencies { androidTestImplementation 'androidx.test.ext:junit:1.1.2' androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' - implementation 'org.pytorch:pytorch_android_lite:1.9.0' - + implementation 'org.pytorch:pytorch_android_lite:1.10.0' } \ No newline at end of file diff --git a/SpeechRecognition/app/src/main/java/org/pytorch/demo/speechrecognition/MainActivity.java b/SpeechRecognition/app/src/main/java/org/pytorch/demo/speechrecognition/MainActivity.java index 10b4537a..9be2c694 100644 --- a/SpeechRecognition/app/src/main/java/org/pytorch/demo/speechrecognition/MainActivity.java +++ b/SpeechRecognition/app/src/main/java/org/pytorch/demo/speechrecognition/MainActivity.java @@ -32,7 +32,7 @@ public class MainActivity extends AppCompatActivity implements Runnable { private static final String TAG = MainActivity.class.getName(); - private Module mModuleEncoder; + private Module module; private TextView mTextView; private Button mButton; @@ -193,8 +193,8 @@ public void run() { } private String recognize(float[] floatInputBuffer) { - if (mModuleEncoder == null) { - mModuleEncoder = LiteModuleLoader.load(assetFilePath(getApplicationContext(), "wav2vec2.ptl")); + if (module == null) { + module = LiteModuleLoader.load(assetFilePath(getApplicationContext(), "wav2vec2.ptl")); } double wav2vecinput[] = new double[RECORDING_LENGTH]; @@ -206,7 +206,7 @@ private String recognize(float[] floatInputBuffer) { inTensorBuffer.put((float)val); Tensor inTensor = Tensor.fromBlob(inTensorBuffer, new long[]{1, RECORDING_LENGTH}); - final String result = mModuleEncoder.forward(IValue.from(inTensor)).toStr(); + final String result = module.forward(IValue.from(inTensor)).toStr(); return result; } diff --git a/TorchVideo/README.md b/TorchVideo/README.md index e7f062d5..47016ea3 100644 --- a/TorchVideo/README.md +++ b/TorchVideo/README.md @@ -6,9 +6,9 @@ The newly released open-sourced [PyTorchVideo](https://github.com/facebookresear ## Prerequisites -* PyTorch 1.9.0, torchvision 0.10.0, PyTorchVideo 0.1.1 (Optional) +* PyTorch 1.10.0, torchvision 0.11.0, PyTorchVideo 0.1.3 (Optional) * Python 3.8 or above (Optional) -* Android library org.pytorch:pytorch_android_lite:1.9.0, org.pytorch:pytorch_android_torchvision:1.9.0 +* Android library org.pytorch:pytorch_android_lite:1.10.0, org.pytorch:pytorch_android_torchvision_lite:1.10.0 * Android Studio 4.0.1 or later ## Quick Start @@ -17,19 +17,17 @@ The newly released open-sourced [PyTorchVideo](https://github.com/facebookresear If you don't have the PyTorch environment set up to run the script, you can download the model file `video_classification.ptl` [here](https://drive.google.com/file/d/1ti8Eb59L5BZV3YJa-c0AUx6XCgEvsqlV/view?usp=sharing) to the `android-demo-app/TorchVideo/app/src/main/assets` folder, then skip the rest of this step and go to step 2 directly. -Be aware that the downloadable model file was created with PyTorch 1.9.0, matching the PyTorch Android library 1.9.0 specified in the project's `build.gradle` file as `implementation 'org.pytorch:pytorch_android_lite:1.9.0'`. If you use a different version of PyTorch to create your model by following the instructions below, make sure you specify the same PyTorch Android library version in the `build.gradle` file to avoid possible errors caused by the version mismatch. Furthermore, if you want to use the latest PyTorch master code to create the model, follow the steps at [Building PyTorch Android from Source](https://pytorch.org/mobile/android/#building-pytorch-android-from-source) and [Using the PyTorch Android Libraries Built](https://pytorch.org/mobile/android/#using-the-pytorch-android-libraries-built-from-source-or-nightly) on how to use the model in Android. - To create the model yourself, simply run the following commands: ``` -conda create -n pt19 python=3.8.5 -conda activate pt19 +conda create -n pt1.10 python=3.8.5 +conda activate pt1.10 pip install torch torchvision pip install pytorchvideo # pip list|grep torch -# torch 1.9.0 -# torchvision 0.10.0 -# pytorchvideo 0.1.1 +# torch 1.10.0 +# torchvision 0.11.1 +# pytorchvideo 0.1.3 cd android-demo-app/TorchVideo python build_model.py @@ -39,7 +37,12 @@ The model file `video_classification.ptl` will be created and saved in the `app/ ### 2. Build with Android Studio -Start Android Studio, then open the project located in `android-demo-app/TorchVideo` +Start Android Studio, then open the project located in `android-demo-app/TorchVideo`. Note the app's `build.gradle` file has the following lines: + +``` +implementation 'org.pytorch:pytorch_android_lite:1.10.0' +implementation 'org.pytorch:pytorch_android_torchvision_lite:1.10.0' +``` ### 3. Run the app diff --git a/TorchVideo/app/build.gradle b/TorchVideo/app/build.gradle index bedbb400..57229a24 100644 --- a/TorchVideo/app/build.gradle +++ b/TorchVideo/app/build.gradle @@ -47,7 +47,6 @@ dependencies { implementation "androidx.camera:camera-core:$camerax_version" implementation "androidx.camera:camera-camera2:$camerax_version" - implementation 'org.pytorch:pytorch_android_lite:1.9.0' - implementation 'org.pytorch:pytorch_android_torchvision:1.9.0' - + implementation 'org.pytorch:pytorch_android_lite:1.10.0' + implementation 'org.pytorch:pytorch_android_torchvision_lite:1.10.0' } \ No newline at end of file diff --git a/ViT4MNIST/README.md b/ViT4MNIST/README.md index 02bf0d44..101480fd 100644 --- a/ViT4MNIST/README.md +++ b/ViT4MNIST/README.md @@ -12,16 +12,16 @@ In this demo app, we'll integrate the two oldest and most popular image datasets ## Prerequisites -* PyTorch 1.7 or later (Optional) +* PyTorch 1.10 or later (Optional) * Python 3.8 (Optional) -* Android Pytorch library 1.7 or later +* Android Pytorch library 1.10 or later * Android Studio 4.0.1 or later ## Quick Start on Using Facebook DeiT ### 1. Prepare the Model (Optional) -To use a pre-trained Facebook DeiT model and convert it to TorchScript, first install [PyTorch](https://pytorch.org/get-started/locally/) 1.7 or later, then install [timm](https://github.com/rwightman/pytorch-image-models) using `pip install timm==0.3.2`, and finally run the following script: +To use a pre-trained Facebook DeiT model and convert it to TorchScript, first install [PyTorch](https://pytorch.org/get-started/locally/) 1.10 or later, then install [timm](https://github.com/rwightman/pytorch-image-models) using `pip install timm==0.3.2`, and finally run the following script: ``` python convert_deit.py @@ -60,14 +60,12 @@ To Test Run the Android ViT4MNIST demo app, follow the steps below: ### 1. Prepare the Model (Optional) -On a Terminal, with PyTorch 1.7.0 and [einops](https://pypi.org/project/einops/) installed, run : +On a Terminal, with PyTorch 1.10.0 and [einops](https://pypi.org/project/einops/) installed (using `pip install einops`), run : ``` python mnist_vit.py ``` -The model definition in `vit_pytorch.py` and training code in `mnist_vit.py` are mostly taken from the blog [here](https://towardsdatascience.com/a-demonstration-of-using-vision-transformers-in-pytorch-mnist-handwritten-digit-recognition-407eafbc15b0). After the training, which takes about 20 minutes on a MacBook Pro, the model is saved as `vit4mnist.pt` and then dynamic-quantized, converted to TorchScript, optimized, and saved as `vit4mnist.pth`, which should be the same as the one already added in the app project, located at `app/src/main/assets`. - -Be aware that the model file was created with PyTorch 1.7.0, matching the PyTorch Android library 1.7.0 specified in the project's `build.gradle` file as `implementation 'org.pytorch:pytorch_android:1.7.0'`. If you use a different version of PyTorch to create your model by following the instructions below, make sure you specify the same PyTorch Android library version in the `build.gradle` file to avoid possible errors caused by the version mismatch. Furthermore, if you want to use the latest PyTorch master code to create the model, follow the steps at [Building PyTorch Android from Source](https://pytorch.org/mobile/android/#building-pytorch-android-from-source) and [Using the PyTorch Android Libraries Built](https://pytorch.org/mobile/android/#using-the-pytorch-android-libraries-built-from-source-or-nightly) on how to use the model in Android. +The model definition in `vit_pytorch.py` and training code in `mnist_vit.py` are mostly taken from the blog [here](https://towardsdatascience.com/a-demonstration-of-using-vision-transformers-in-pytorch-mnist-handwritten-digit-recognition-407eafbc15b0). After the training, which takes about 20 minutes on a MacBook Pro, the model is saved as `vit4mnist.pt` and then dynamic-quantized, converted to TorchScript, optimized, and saved as `vit4mnist.pth`, which should be the same as the one already added in the app project, located at `app/src/main/assets`. ### 2. Build and run with Android Studio diff --git a/ViT4MNIST/app/build.gradle b/ViT4MNIST/app/build.gradle index 52fc8280..ade78186 100644 --- a/ViT4MNIST/app/build.gradle +++ b/ViT4MNIST/app/build.gradle @@ -33,6 +33,6 @@ dependencies { androidTestImplementation 'androidx.test.ext:junit:1.1.2' androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' - implementation 'org.pytorch:pytorch_android:1.7.0' - implementation 'org.pytorch:pytorch_android_torchvision:1.7.0' + implementation 'org.pytorch:pytorch_android_lite:1.10.0' + implementation 'org.pytorch:pytorch_android_torchvision_lite:1.10.0' } \ No newline at end of file diff --git a/ViT4MNIST/app/src/main/assets/vit4mnist.pth b/ViT4MNIST/app/src/main/assets/vit4mnist.pth deleted file mode 100644 index 843029bf..00000000 Binary files a/ViT4MNIST/app/src/main/assets/vit4mnist.pth and /dev/null differ diff --git a/ViT4MNIST/app/src/main/assets/vit4mnist.ptl b/ViT4MNIST/app/src/main/assets/vit4mnist.ptl new file mode 100644 index 00000000..9c8f6e42 Binary files /dev/null and b/ViT4MNIST/app/src/main/assets/vit4mnist.ptl differ diff --git a/ViT4MNIST/app/src/main/java/org/pytorch/demo/vit4mnist/MainActivity.java b/ViT4MNIST/app/src/main/java/org/pytorch/demo/vit4mnist/MainActivity.java index ce4d7dd5..e2f64656 100644 --- a/ViT4MNIST/app/src/main/java/org/pytorch/demo/vit4mnist/MainActivity.java +++ b/ViT4MNIST/app/src/main/java/org/pytorch/demo/vit4mnist/MainActivity.java @@ -12,7 +12,9 @@ package org.pytorch.demo.vit4mnist; +import android.content.Context; import android.os.Bundle; +import android.util.Log; import android.util.Pair; import android.view.View; import android.widget.Button; @@ -21,10 +23,15 @@ import androidx.appcompat.app.AppCompatActivity; import org.pytorch.IValue; +import org.pytorch.LiteModuleLoader; import org.pytorch.Module; -import org.pytorch.PyTorchAndroid; import org.pytorch.Tensor; +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.FloatBuffer; import java.util.List; @@ -43,6 +50,26 @@ public class MainActivity extends AppCompatActivity implements Runnable { private static final float NON_BLANK = (1.0f - MNISI_STD) / MNISI_MEAN; private static final int MNIST_IMAGE_SIZE = 28; + + public static String assetFilePath(Context context, String assetName) throws IOException { + File file = new File(context.getFilesDir(), assetName); + if (file.exists() && file.length() > 0) { + return file.getAbsolutePath(); + } + + try (InputStream is = context.getAssets().open(assetName)) { + try (OutputStream os = new FileOutputStream(file)) { + byte[] buffer = new byte[4 * 1024]; + int read; + while ((read = is.read(buffer)) != -1) { + os.write(buffer, 0, read); + } + os.flush(); + } + return file.getAbsolutePath(); + } + } + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -67,7 +94,12 @@ public void onClick(View v) { } }); - mModule = PyTorchAndroid.loadModuleFromAsset(getAssets(), "vit4mnist.pth"); + try { + mModule = LiteModuleLoader.load(MainActivity.assetFilePath(getApplicationContext(), "vit4mnist.ptl")); + } catch (IOException e) { + Log.e("VIT4MNIST", "Error reading assets", e); + finish(); + } } public void run() { diff --git a/ViT4MNIST/mnist_vit.py b/ViT4MNIST/mnist_vit.py index 572bca7d..1f714e56 100644 --- a/ViT4MNIST/mnist_vit.py +++ b/ViT4MNIST/mnist_vit.py @@ -98,4 +98,4 @@ def evaluate(model, data_loader, loss_history): ts_model = torch.jit.trace(quantized_model, dummy_input) optimized_torchscript_model = optimize_for_mobile(ts_model) # the quantized, scripted, and optimized model -optimized_torchscript_model.save("vit4mnist.pth") +optimized_torchscript_model._save_for_lite_interpreter("app/src/main/assets/vit4mnist.ptl")