remove ftfy imports and add warning if not installed (microsoft#578)

Co-authored-by: Sayan Shaw <sayanshaw@microsoft.com>

requirements-dev.txt

@@ -1,7 +1,6 @@
 # include requirements.txt so pip has context to avoid installing incompatible dependencies
 -r requirements.txt
 pytest
-ftfy
 # multiple versions of onnxruntime are supported, but only one can be installed at a time
 protobuf         < 4.0.0
 onnxruntime      >=1.12.0

test/test_autotokenizer.py

@@ -1,9 +1,9 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 import unittest
+import pkg_resources
 
 import numpy as np
-import ftfy
 from transformers import AutoTokenizer, GPT2Tokenizer
 from onnxruntime_extensions import OrtPyFunction, gen_processing_models, ort_inference, util
 
@@ -129,4 +129,8 @@ def print_prime(n):
 
 
 if __name__ == '__main__':
+    try:
+        dist = pkg_resources.get_distribution('ftfy')
+    except pkg_resources.DistributionNotFound:
+        raise Exception("WARNING: ftfy is not installed - it is required for parity between CLIPTokenizer and CLIPTokenizerFast.")
     unittest.main()

test/test_cliptok.py

@@ -1,7 +1,7 @@
 import unittest
 import numpy as np
 import onnxruntime as _ort
-import ftfy
+import pkg_resources
 
 from pathlib import Path
 from onnx import helper, onnx_pb as onnx_proto
@@ -150,4 +150,8 @@ def test_optional_outputs(self):
 
 
 if __name__ == "__main__":
+    try:
+        dist = pkg_resources.get_distribution('ftfy')
+    except pkg_resources.DistributionNotFound:
+        raise Exception("WARNING: ftfy is not installed - it is required for parity between CLIPTokenizer and CLIPTokenizerFast.")
     unittest.main()