From 92d17b0a5258bcc254a82ce634071df2f5564c05 Mon Sep 17 00:00:00 2001
From: Sayan Shaw <52221015+sayanshaw24@users.noreply.github.com>
Date: Thu, 19 Oct 2023 14:09:55 -0700
Subject: [PATCH] remove ftfy imports and add warning if not installed (#578)

Co-authored-by: Sayan Shaw <sayanshaw@microsoft.com>
---
 requirements-dev.txt       | 1 -
 test/test_autotokenizer.py | 6 +++++-
 test/test_cliptok.py       | 6 +++++-
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/requirements-dev.txt b/requirements-dev.txt
index ffd23596f..d6ad1fe49 100644
--- a/requirements-dev.txt
+++ b/requirements-dev.txt
@@ -1,7 +1,6 @@
 # include requirements.txt so pip has context to avoid installing incompatible dependencies
 -r requirements.txt
 pytest
-ftfy
 # multiple versions of onnxruntime are supported, but only one can be installed at a time
 protobuf         < 4.0.0
 onnxruntime      >=1.12.0
diff --git a/test/test_autotokenizer.py b/test/test_autotokenizer.py
index f9a388360..1036f0a9a 100644
--- a/test/test_autotokenizer.py
+++ b/test/test_autotokenizer.py
@@ -1,9 +1,9 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 import unittest
+import pkg_resources
 
 import numpy as np
-import ftfy
 from transformers import AutoTokenizer, GPT2Tokenizer
 from onnxruntime_extensions import OrtPyFunction, gen_processing_models, ort_inference, util
 
@@ -129,4 +129,8 @@ def print_prime(n):
 
 
 if __name__ == '__main__':
+    try:
+        dist = pkg_resources.get_distribution('ftfy')
+    except pkg_resources.DistributionNotFound:
+        raise Exception("WARNING: ftfy is not installed - it is required for parity between CLIPTokenizer and CLIPTokenizerFast.")
     unittest.main()
diff --git a/test/test_cliptok.py b/test/test_cliptok.py
index 16f0737da..77fb1173c 100644
--- a/test/test_cliptok.py
+++ b/test/test_cliptok.py
@@ -1,7 +1,7 @@
 import unittest
 import numpy as np
 import onnxruntime as _ort
-import ftfy
+import pkg_resources
 
 from pathlib import Path
 from onnx import helper, onnx_pb as onnx_proto
@@ -150,4 +150,8 @@ def test_optional_outputs(self):
 
 
 if __name__ == "__main__":
+    try:
+        dist = pkg_resources.get_distribution('ftfy')
+    except pkg_resources.DistributionNotFound:
+        raise Exception("WARNING: ftfy is not installed - it is required for parity between CLIPTokenizer and CLIPTokenizerFast.")
     unittest.main()