Merge pull request #138 from uTensor/remove-legacy

Remove legacy

.circleci/config.yml

@@ -7,20 +7,20 @@ jobs:
       - checkout
         #- run: apt update
       - run: mkdir -p /tmp/coverage && mkdir -p /tmp/test_results
-      - run: pip3 install pytest
+      - run: pip3 install -U pip && pip3 install pytest
       - run: pip3 install .
-      - run: 
+      - run:
           name: Test
-          command: LC_ALL=C.UTF-8 LANG=C.UTF-8 pytest -vv tests -m 'not slow_test and not deprecated'
+          command: LC_ALL=C.UTF-8 LANG=C.UTF-8 pytest -vv tests -m 'not slow_test and not deprecated and not beta_release'
           #- run: genhtml main_coverage.info --output-directory /tmp/coverage
           #- store_test_results:
           #    path: /tmp/test_results
           #- store_artifacts:
           #    path: /tmp/coverage
           #    prefix: coverage
-        
+
 workflows:
-    version: 2
-    run_ci:
-        jobs:
-            - build
+  version: 2
+  run_ci:
+    jobs:
+      - build

Makefile

@@ -3,13 +3,13 @@
 tests:
 	rm -f tests_log.txt
 	make test_utils test_ir test_transformer test_frontend \
-		test_matcher test_graph_constructor test_backend
+		test_graph_constructor test_backend test_matcher test_api
 
 test_%:
 	@if [ -d .venv ]; then \
-		 pipenv run pytest tests/$@ -m 'not slow_test and not deprecated' -vv -s | tee -a tests_log.txt; \
+		 pipenv run pytest tests/$@ -m 'not slow_test and not deprecated and not beta_release' -vv -s | tee -a tests_log.txt; \
 	 else \
-	 	 pytest tests/$@ -vv -s -m 'not slow_test and not deprecated' | tee -a tests_log.txt; \
+	 	 pytest tests/$@ -vv -s -m 'not slow_test and not deprecated and not beta_release' | tee -a tests_log.txt; \
 	 fi;
 
 package:
@@ -29,6 +29,6 @@ upload: package
 
 clean:
 	rm -rf tests_log.txt *.pdf \
-	models data \
+	models constants data \
 	tests/test_backend/{models,data} \
 	.pytest_cache dist/ build/

example_config.toml

@@ -1,29 +1,21 @@
+# utensor-cli version 1.0.0
 # https://github.com/toml-lang/toml
 # <target_name>.<component>.<part>
 # we use string 'None' to represent python None value
 # you should convert the string to None if you try to write extension for utensor_cgen
-[utensor.backend]
-legacy-api = true
-
 [utensor.backend.tensor_alloc_planner]
 max_pool_size = 1048576
 include_inputs = false
+include_outputs = true
 out_fname = "None"
 enabled = true
-
-[utensor.backend.legacy_code_generator]
-src_fname = "None"
-params_dir = "data"
-embed_params_dir = "/fs/data"
-model_dir = "models"
-debug_cmt = false
-
-[utensor.backend.legacy_graph_lower]
+data_alignment = 2
 
 [utensor.backend.rearch_code_generator]
+use_model_api = true
 src_fname = "None"
 header_fname = "None"
-params_dir = "data"
+params_dir = "constants"
 model_dir = "models"
 meta_data_pool_size = "auto"
 ram_data_pool_size = "auto"
@@ -32,7 +24,7 @@ ram_data_pool_size = "auto"
 
 [utensor.backend.pipeline_transformer]
 save_graph = false
-transform_methods = [ "dropout(name_pattern=r'(dropout[_\\w\\d]*)/.*')", "linear_reorder", "quantize", "conv_pool", "inline", "biasAdd", "remove_id_op", "fake_gather_v2", "refcnt",]
+transform_methods = [ "dropout(name_pattern=r'(dropout[_\\w\\d]*)/.*')", "inline", "biasAdd", "remove_id_op", "refcnt",]
 
 [utensor.backend.tensor_alloc_planner.aesthetic_kwargs]
 split_on_large_graph = true

pyproject.toml

@@ -0,0 +1,3 @@
+[build-system]
+requires = ["setuptools", "wheel"]
+build-backend = "setuptools.build_meta:__legacy__"

setup.py

@@ -45,7 +45,7 @@ class _Develop(_CompileFlatbuffMixin, _develop):
     entry_points={"console_scripts": ["utensor-cli=utensor_cgen.cli:cli"]},
     install_requires=[
         "Jinja2",
-        "tensorflow>=2.1.0",
+        "tensorflow>=2.3.0",
         "onnx",
         "keras2onnx",
         "idx2numpy",

tests/test_api/test_keras_onnx/conftest.py

@@ -1,16 +1,16 @@
-from pytest import fixture
-
 import os
+
 import numpy as np
+from pytest import fixture
 
-import tensorflow as tf
-from tensorflow.keras.models import Sequential, save_model, load_model
-from tensorflow.keras.layers import MaxPool2D, ReLU, Conv2D, Softmax, Dense, Flatten
-from tensorflow.keras.losses import sparse_categorical_crossentropy
-from tensorflow.keras.optimizers import Adam
 
 @fixture(scope='session', name='keras_model')
 def keras_model():
+    from tensorflow.keras.layers import (Conv2D, Dense, Flatten, MaxPool2D,
+                                         ReLU, Softmax)
+    from tensorflow.keras.losses import sparse_categorical_crossentropy
+    from tensorflow.keras.models import Sequential, load_model, save_model
+    from tensorflow.keras.optimizers import Adam
 
     input_shape = (28,28,1)
     no_classes = 10
@@ -38,6 +38,7 @@ def keras_model():
 
 @fixture(scope='session', name='keras_model_dset')
 def keras_model_dset():
+    import tensorflow as tf
     mnist = tf.keras.datasets.mnist
 
     (x_train, y_train), (x_test, y_test) = mnist.load_data()

tests/test_api/test_keras_onnx/test_export.py

@@ -3,9 +3,11 @@
 from pathlib import Path
 
 import pytest
+
 import utensor_cgen.api.export as export
-import tensorflow.keras as keras
 
+
+@pytest.mark.beta_release
 def test_keras_model(keras_model, keras_model_dset):
 
     assert keras_model, 'Keras Model generation failed'
@@ -17,8 +19,10 @@ def test_keras_model(keras_model, keras_model_dset):
         target='utensor'
     )
 
-
+@pytest.mark.beta_release
 def test_keras_model_path(keras_model, keras_model_dset):
+  import tensorflow.keras as keras
+
   with tempfile.TemporaryDirectory(prefix='utensor_') as tmp_dir:
     dir_path = Path(tmp_dir)
     keras_model_path = os.path.join(dir_path, 'model.h5')
@@ -32,4 +36,4 @@ def test_keras_model_path(keras_model, keras_model_dset):
         representive_dataset=keras_model_dset,
         model_name='model',
         target='utensor'
-    )
+    )

tests/test_api/test_keras_tflite/conftest.py

@@ -1,16 +1,16 @@
-from pytest import fixture
-
 import os
+
 import numpy as np
+from pytest import fixture
 
-import tensorflow as tf
-from tensorflow.keras.models import Sequential, save_model, load_model
-from tensorflow.keras.layers import MaxPool2D, ReLU, Conv2D, Softmax, Dense, Flatten
-from tensorflow.keras.losses import sparse_categorical_crossentropy
-from tensorflow.keras.optimizers import Adam
 
-@fixture(scope='session', name='keras_model')
+@fixture(scope='function', name='keras_model')
 def keras_model():
+    from tensorflow.keras.layers import (Conv2D, Dense, Flatten, MaxPool2D,
+                                         ReLU, Softmax)
+    from tensorflow.keras.losses import sparse_categorical_crossentropy
+    from tensorflow.keras.models import Sequential, load_model, save_model
+    from tensorflow.keras.optimizers import Adam
 
     input_shape = (28,28,1)
     no_classes = 10
@@ -37,27 +37,27 @@ def keras_model():
 
     return model
 
-@fixture(scope='session', name='keras_model_dset')
+@fixture(scope='function', name='keras_model_dset')
 def keras_model_dset():
+    import tensorflow as tf
+
     mnist = tf.keras.datasets.mnist
 
-    (x_train, y_train), (x_test, y_test) = mnist.load_data()
+    (x_train, _), (x_test, _) = mnist.load_data()
     x_train, x_test = x_train / 255.0, x_test / 255.0
 
     # Add a channels dimension
     x_train = x_train[..., tf.newaxis]
     x_test = x_test[..., tf.newaxis]
 
     num_calibration_steps = 128
-    calibration_dtype = tf.float32
-    input_shape = (28,28,1)
+    calibration_dtype = 'float32'
 
     def representative_dataset_gen():
         for _ in range(num_calibration_steps):
             rand_idx = np.random.randint(0, x_test.shape[0]-1)
             sample = x_test[rand_idx]
-            sample = sample[tf.newaxis, ...]
-            sample = tf.cast(sample, dtype=calibration_dtype)
+            sample = sample[tf.newaxis, ...].astype(calibration_dtype)
             yield [sample]
 
     return representative_dataset_gen

tests/test_api/test_keras_tflite/test_export.py

@@ -3,8 +3,9 @@
 from pathlib import Path
 
 import pytest
+
 import utensor_cgen.api.export as export
-import tensorflow.keras as keras
+
 
 def test_keras_model(keras_model, keras_model_dset):
   assert keras_model, 'Keras Model generation failed'
@@ -16,7 +17,10 @@ def test_keras_model(keras_model, keras_model_dset):
     target='utensor'
   )
 
+
 def test_keras_model_path(keras_model, keras_model_dset):
+  import tensorflow.keras as keras
+
   with tempfile.TemporaryDirectory(prefix='utensor_') as tmp_dir:
     dir_path = Path(tmp_dir)
     keras_model_path = os.path.join(dir_path, 'model')
@@ -31,4 +35,4 @@ def test_keras_model_path(keras_model, keras_model_dset):
       representive_dataset=keras_model_dset,
       model_name='model',
       target='utensor'
-    )
+    )

tests/test_backend/test_utensor.py

@@ -3,24 +3,6 @@
 import pytest
 
 
-def test_legacy_utensor(mlp_ugraph):
-    from utensor_cgen.backend.utensor import uTensorBackend
-
-    this_dir = os.path.dirname(__file__)
-
-    backend = uTensorBackend(config={
-        'utensor': {
-            'backend': {
-                'legacy-api': True,
-                'legacy_code_generator': {
-                    'model_dir': os.path.join(this_dir, 'models'),
-                    'params_dir': os.path.join(this_dir, 'data'),
-                },
-            },
-        }
-    })
-    backend.apply(mlp_ugraph)
-
 def test_rearch_utensor(simple_ugraph):
     from utensor_cgen.backend.utensor import uTensorBackend
 
@@ -29,7 +11,6 @@ def test_rearch_utensor(simple_ugraph):
     backend = uTensorBackend(config={
         'utensor': {
             'backend': {
-                'legacy-api': False,
                 'rearch_code_generator': {
                     'model_dir': os.path.join(this_dir, 'models'),
                     'params_dir': os.path.join(this_dir, 'data'),

tests/test_graph_constructor/test_ctx.py

@@ -8,11 +8,16 @@ def test_ugraph_build_ctx():
     ugraph = uTensorGraph()
 
     with ugraph.begin_construction():
-        ugraph.add_op(op_type='Const', name='x', value=np.array([1]), is_output=True)
+        ugraph.add_op(
+            'Constant',
+            name='x',
+            values=np.array([1]),
+            is_output=True
+        )
 
     assert ugraph.is_finalized
     assert ugraph.output_nodes == ['x']
     try:
-        ugraph.add_op(op_type='Const', name='y', value=np.array([2]))
+        ugraph.add_op('Constant', name='y', values=np.array([2]))
     except GraphFinalizedError:
         pass

tests/test_graph_constructor/test_ops/test_op_add.py

@@ -4,19 +4,19 @@
 def test_op_add(ugraph):
     with ugraph.begin_construction():
         tensor_x, = ugraph.add_op(
-            op_type='Const',
+            op_type='Constant',
             name='x',
-            value=np.random.rand(1, 3, 5).astype('float32')
+            values=np.random.rand(1, 3, 5).astype('float32')
         )
         tensor_y, = ugraph.add_op(
-            op_type='Const',
+            op_type='Constant',
             name='y',
-            value=np.random.rand(1, 3, 5).astype('float32')
+            values=np.random.rand(1, 3, 5).astype('float32')
         )
         tensor_z, = ugraph.add_op(
+            'AddOperator',
             tensor_x,
             tensor_y,
-            op_type='Add',
             name='z',
             is_output=True
         )

tests/test_graph_constructor/test_ops/test_op_argmax.py

@@ -1,6 +1,8 @@
 import numpy as np
+import pytest
 
 
+@pytest.mark.deprecated
 def test_op_argmax(ugraph):
     with ugraph.begin_construction():
         tensor_logits, = ugraph.add_op(

tests/test_graph_constructor/test_ops/test_op_const.py

@@ -4,9 +4,9 @@
 def test_op_const(ugraph):
     with ugraph.begin_construction():
         out_tensor, = ugraph.add_op(
-            op_type='Const',
+            op_type='Constant',
             name='ones',
-            value=np.ones((3, 3), dtype=np.dtype('float32')),
+            values=np.ones((3, 3), dtype=np.dtype('float32')),
             is_output=True
         )
 

tests/test_graph_constructor/test_ops/test_op_conv2d.py

@@ -4,20 +4,20 @@
 def test_op_conv2d(ugraph):
     with ugraph.begin_construction():
         tensor_x, = ugraph.add_op(
-            np.random.rand(10, 512, 512, 5),
-            op_type='Const',
+            'Constant',
+            values=np.random.rand(10, 512, 512, 5),
             name='feature_map'
         )
         tensor_w, = ugraph.add_op(
-            np.random.rand(32, 32, 5, 10),
-            op_type='Const',
+            'Constant',
+            values=np.random.rand(32, 32, 5, 10),
             name='filter'
         )
         out, = ugraph.add_op(
+            'Conv2dOperator',
             tensor_x,
             tensor_w,
             name='output',
-            op_type='Conv2D',
             padding='SAME',
             stride_height=2,
             stride_width=2,