Support React Native

package.json

@@ -38,12 +38,18 @@
   },
   "homepage": "https://github.com/xenova/transformers.js#readme",
   "dependencies": {
-    "onnxruntime-web": "1.14.0",
-    "sharp": "^0.32.0",
-    "@huggingface/jinja": "^0.2.2"
+    "@huggingface/jinja": "^0.2.2",
+    "image-codecs": "^0.1.0",
+    "node-wav": "^0.0.2",
+    "onnxruntime-web": "^1.17.0",
+    "sharp": "^0.32.0"
   },
   "optionalDependencies": {
-    "onnxruntime-node": "1.14.0"
+    "browserify-zlib": "^0.2.0",
+    "onnxruntime-node": "^1.17.0",
+    "onnxruntime-react-native": "^1.17.0",
+    "path-browserify": "^1.0.1",
+    "react-native-fs": "^2.20.0"
   },
   "devDependencies": {
     "@types/jest": "^29.5.1",
@@ -59,6 +65,7 @@
     "webpack-dev-server": "^4.13.3"
   },
   "overrides": {
+    "jpeg-js": "^0.4.4",
     "semver": "^7.5.4",
     "protobufjs": "^7.2.6"
   },
@@ -69,8 +76,18 @@
     "README.md",
     "LICENSE"
   ],
+  "react-native": {
+    "fs": "react-native-fs",
+    "onnxruntime-web": false,
+    "onnxruntime-node": "onnxruntime-react-native",
+    "image-codecs": "image-codecs",
+    "path": "path-browserify",
+    "stream/web": false,
+    "zlib": "browserify-zlib"
+  },
   "browser": {
     "fs": false,
+    "image-codecs": false,
     "path": false,
     "url": false,
     "sharp": false,

src/backends/onnx.js

@@ -29,7 +29,15 @@ export const executionProviders = [
     'wasm'
 ];
 
-if (typeof process !== 'undefined' && process?.release?.name === 'node') {
+if (typeof navigator !== 'undefined' && navigator.product === 'ReactNative') {
+    // Running in a RN or node-like environment.
+    ONNX = ONNX_NODE.default ?? ONNX_NODE;
+
+    // Add `cpu` execution provider, and remove `wasm`.
+    executionProviders.unshift('cpu');
+    executionProviders.splice(executionProviders.indexOf('wasm'), 1);
+
+} else if (typeof process !== 'undefined' && process?.release?.name === 'node') {
     // Running in a node-like environment.
     ONNX = ONNX_NODE.default ?? ONNX_NODE;
 
@@ -47,4 +55,4 @@ if (typeof process !== 'undefined' && process?.release?.name === 'node') {
     if (isIOS) {
         ONNX.env.wasm.simd = false;
     }
-}
+}

src/env.js

@@ -25,41 +25,47 @@
 import fs from 'fs';
 import path from 'path';
 import url from 'url';
+import { Buffer } from 'buffer';
 
 import { ONNX } from './backends/onnx.js';
 const { env: onnx_env } = ONNX;
 
 const VERSION = '2.17.2';
 
 // Check if various APIs are available (depends on environment)
+const IS_REACT_NATIVE = typeof navigator !== 'undefined' && navigator.product === 'ReactNative';
 const WEB_CACHE_AVAILABLE = typeof self !== 'undefined' && 'caches' in self;
-const FS_AVAILABLE = !isEmpty(fs); // check if file system is available
+const FS_AVAILABLE = !isEmpty(fs) || IS_REACT_NATIVE; // check if file system is available
 const PATH_AVAILABLE = !isEmpty(path); // check if path is available
 
 const RUNNING_LOCALLY = FS_AVAILABLE && PATH_AVAILABLE;
 
-const __dirname = RUNNING_LOCALLY
-    ? path.dirname(path.dirname(url.fileURLToPath(import.meta.url)))
-    : './';
+
+let localPath = './';
+if (IS_REACT_NATIVE) {
+    localPath = fs.DocumentDirectoryPath;
+} else if (RUNNING_LOCALLY) {
+    localPath = path.dirname(path.dirname(url.fileURLToPath(import.meta.url)));
+}
 
 // Only used for environments with access to file system
 const DEFAULT_CACHE_DIR = RUNNING_LOCALLY
-    ? path.join(__dirname, '/.cache/')
+    ? path.join(localPath, '/.cache/')
     : null;
 
 // Set local model path, based on available APIs
 const DEFAULT_LOCAL_MODEL_PATH = '/models/';
 const localModelPath = RUNNING_LOCALLY
-    ? path.join(__dirname, DEFAULT_LOCAL_MODEL_PATH)
+    ? path.join(localPath, DEFAULT_LOCAL_MODEL_PATH)
     : DEFAULT_LOCAL_MODEL_PATH;
 
-if (onnx_env?.wasm) {
+if (!IS_REACT_NATIVE && onnx_env?.wasm) {
     // Set path to wasm files. This is needed when running in a web worker.
     // https://onnxruntime.ai/docs/api/js/interfaces/Env.WebAssemblyFlags.html#wasmPaths
     // We use remote wasm files by default to make it easier for newer users.
     // In practice, users should probably self-host the necessary .wasm files.
     onnx_env.wasm.wasmPaths = RUNNING_LOCALLY
-        ? path.join(__dirname, '/dist/')
+        ? path.join(localPath, '/dist/')
         : `https://cdn.jsdelivr.net/npm/@xenova/transformers@${VERSION}/dist/`;
 }
 
@@ -92,9 +98,11 @@ export const env = {
 
         // TensorFlow.js
         tfjs: {},
+
+        Uint8Array: IS_REACT_NATIVE ? Buffer : Uint8Array,
     },
 
-    __dirname,
+    __dirname: localPath,
     version: VERSION,
 
     /////////////////// Model settings ///////////////////
@@ -106,6 +114,8 @@ export const env = {
     localModelPath: localModelPath,
     useFS: FS_AVAILABLE,
 
+    rnUseCanvas: true,
+
     /////////////////// Cache settings ///////////////////
     useBrowserCache: WEB_CACHE_AVAILABLE,
 

src/models.js

@@ -51,6 +51,7 @@ import {
 
 import {
     getModelFile,
+    getModelPath,
     getModelJSON,
 } from './utils/hub.js';
 
@@ -81,12 +82,16 @@ import {
     Tensor,
 } from './utils/tensor.js';
 
+import { env } from './env.js';
+
 import { executionProviders, ONNX } from './backends/onnx.js';
 import { medianFilter } from './transformers.js';
-const { InferenceSession, Tensor: ONNXTensor, env } = ONNX;
+const { InferenceSession, Tensor: ONNXTensor, env: onnx_env } = ONNX;
 
 /** @typedef {import('onnxruntime-web').InferenceSession} InferenceSession */
 
+const IS_BROWSER = typeof navigator !== 'undefined' && navigator.product !== 'ReactNative';
+
 //////////////////////////////////////////////////
 // Model types: used internally
 const MODEL_TYPES = {
@@ -119,27 +124,20 @@ const MODEL_CLASS_TO_NAME_MAPPING = new Map();
  */
 async function constructSession(pretrained_model_name_or_path, fileName, options) {
     // TODO add option for user to force specify their desired execution provider
-    let modelFileName = `onnx/${fileName}${options.quantized ? '_quantized' : ''}.onnx`;
-    let buffer = await getModelFile(pretrained_model_name_or_path, modelFileName, true, options);
 
-    try {
-        return await InferenceSession.create(buffer, {
-            executionProviders,
-        });
-    } catch (err) {
-        // If the execution provided was only wasm, throw the error
-        if (executionProviders.length === 1 && executionProviders[0] === 'wasm') {
-            throw err;
-        }
+    const sessionOptions = { executionProviders, ...options.session_options };
+    const modelFileName = `onnx/${fileName}${options.quantized ? '_quantized' : ''}.onnx`;
 
-        console.warn(err);
-        console.warn(
-            'Something went wrong during model construction (most likely a missing operation). ' +
-            'Using `wasm` as a fallback. '
-        )
-        return await InferenceSession.create(buffer, {
-            executionProviders: ['wasm']
-        });
+    if (IS_BROWSER || !env.useFS || !env.allowLocalModels || env.useCustomCache) {
+        let buffer = await getModelFile(pretrained_model_name_or_path, modelFileName, true, options);
+
+        return await InferenceSession.create(buffer, sessionOptions);
+    } else {
+        let modelPath = await getModelPath(pretrained_model_name_or_path, modelFileName, true, options);
+        // Download optional external data file
+        await getModelPath(pretrained_model_name_or_path, `${modelFileName}_data`, false, options);
+
+        return await InferenceSession.create(modelPath, sessionOptions);
     }
 }
 
@@ -167,10 +165,10 @@ function validateInputs(session, inputs) {
             missingInputs.push(inputName);
             continue;
         }
-        // NOTE: When `env.wasm.proxy is true` the tensor is moved across the Worker
+        // NOTE: When `onnx_env.wasm.proxy is true` the tensor is moved across the Worker
         // boundary, transferring ownership to the worker and invalidating the tensor.
         // So, in this case, we simply sacrifice a clone for it.
-        checkedInputs[inputName] = env.wasm.proxy ? tensor.clone() : tensor;
+        checkedInputs[inputName] = onnx_env.wasm.proxy ? tensor.clone() : tensor;
     }
     if (missingInputs.length > 0) {
         throw new Error(
@@ -210,7 +208,14 @@ async function sessionRun(session, inputs) {
     } catch (e) {
         // This usually occurs when the inputs are of the wrong type.
         console.error(`An error occurred during model execution: "${e}".`);
-        console.error('Inputs given to model:', checkedInputs);
+        // Not log full data, it may cause crash in React Native
+        console.error(
+            'Inputs given to model:',
+            IS_BROWSER ? checkedInputs : Object.fromEntries(
+                Object.entries(checkedInputs)
+                    .map(([key, tensor]) => [key, { dims: tensor.dims, type: tensor.type }])
+            )
+        );
         throw e;
     }
 }
@@ -224,7 +229,7 @@ async function sessionRun(session, inputs) {
 function replaceTensors(obj) {
     for (let prop in obj) {
         if (obj[prop] instanceof ONNXTensor) {
-            obj[prop] = new Tensor(obj[prop]);
+            obj[prop] = Tensor.fromONNX(obj[prop]);
         } else if (typeof obj[prop] === 'object') {
             replaceTensors(obj[prop]);
         }
@@ -741,6 +746,7 @@ export class PreTrainedModel extends Callable {
         local_files_only = false,
         revision = 'main',
         model_file_name = null,
+        session_options = {},
     } = {}) {
 
         let options = {
@@ -751,6 +757,7 @@ export class PreTrainedModel extends Callable {
             local_files_only,
             revision,
             model_file_name,
+            session_options,
         }
 
         const modelName = MODEL_CLASS_TO_NAME_MAPPING.get(this);
@@ -5524,6 +5531,7 @@ export class PretrainedMixin {
         local_files_only = false,
         revision = 'main',
         model_file_name = null,
+        session_options = {},
     } = {}) {
 
         let options = {
@@ -5534,6 +5542,7 @@ export class PretrainedMixin {
             local_files_only,
             revision,
             model_file_name,
+            session_options,
         }
         config = await AutoConfig.from_pretrained(pretrained_model_name_or_path, options);
         if (!options.config) {

src/pipelines.js

@@ -70,6 +70,9 @@ import {
     quantize_embeddings,
 } from './utils/tensor.js';
 import { RawImage } from './utils/image.js';
+import {
+    fetchBinary
+} from './utils/hub.js';
 
 
 /**
@@ -2678,7 +2681,7 @@ export class TextToAudioPipeline extends (/** @type {new (options: TextToAudioPi
         if (typeof speaker_embeddings === 'string' || speaker_embeddings instanceof URL) {
             // Load from URL with fetch
             speaker_embeddings = new Float32Array(
-                await (await fetch(speaker_embeddings)).arrayBuffer()
+                await (await fetchBinary(speaker_embeddings)).arrayBuffer()
             );
         }
 
@@ -3178,6 +3181,7 @@ export async function pipeline(
         cache_dir = null,
         local_files_only = false,
         revision = 'main',
+        session_options = {},
         model_file_name = null,
     } = {}
 ) {
@@ -3206,6 +3210,7 @@ export async function pipeline(
         cache_dir,
         local_files_only,
         revision,
+        session_options,
         model_file_name,
     }
 
@@ -3251,7 +3256,9 @@ async function loadItems(mapping, model, pretrainedOptions) {
         if (Array.isArray(cls)) {
             promise = new Promise(async (resolve, reject) => {
                 let e;
-                for (let c of cls) {
+                for (let i in cls) {
+                    let isLast = i == cls.length - 1;
+                    let c = cls[i];
                     if (c === null) {
                         // If null, we resolve it immediately, meaning the relevant
                         // class was not found, but it is optional.
@@ -3262,7 +3269,9 @@ async function loadItems(mapping, model, pretrainedOptions) {
                         resolve(await c.from_pretrained(model, pretrainedOptions));
                         return;
                     } catch (err) {
-                        e = err;
+                        if (!isLast || !err?.message.startsWith('Unsupported model type')) {
+                            e = err;
+                        }
                     }
                 }
                 reject(e);

src/tokenizers.js

@@ -35,6 +35,8 @@ import {
 import { max, min, round } from './utils/maths.js';
 import { Tensor } from './utils/tensor.js';
 
+import { env } from './env.js';
+
 import {
     PriorityQueue,
     TokenLattice,
@@ -43,6 +45,8 @@ import {
 
 import { Template } from '@huggingface/jinja';
 
+const { backends: { Uint8Array } } = env;
+
 
 /**
  * @typedef {Object} TokenizerProperties Additional tokenizer-specific properties.
@@ -2047,7 +2051,7 @@ class ByteLevelDecoder extends Decoder {
      */
     convert_tokens_to_string(tokens) {
         const text = tokens.join('');
-        const byteArray = new Uint8Array([...text].map(c => this.byte_decoder[c]));
+        const byteArray = Uint8Array.from([...text].map(c => this.byte_decoder[c]));
         const decoded_text = this.text_decoder.decode(byteArray);
         return decoded_text;
     }