diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index a7eef74fb6a014..a7cedc9af2695d 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -11,7 +11,7 @@ extensions/back/__init__.py extensions/back/AvgPool.py extensions/back/blob_normalizer.py extensions/back/CellNormalizer.py -extensions/back/ChangeCastOutputType.py +extensions/back/ChangeOutputTypeAttributes.py extensions/back/ClampNormalizer.py extensions/back/compress_quantized_weights.py extensions/back/ConvolutionNormalizer.py diff --git a/model-optimizer/extensions/back/ChangeCastOutputType.py b/model-optimizer/extensions/back/ChangeCastOutputType.py deleted file mode 100644 index 976b6b50a29136..00000000000000 --- a/model-optimizer/extensions/back/ChangeCastOutputType.py +++ /dev/null @@ -1,43 +0,0 @@ -# Copyright (C) 2018-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import logging as log - -import numpy as np - -from mo.back.replacement import BackReplacementPattern -from mo.graph.graph import Graph -from mo.middle.passes.convert_data_type import data_type_str_to_np - - -class ChangeCastOutputType(BackReplacementPattern): - """ - Change the Cast dst_type from fp64 to fp32 since not all plugins support fp64 data type. - Change the Cast dst_type from fp32 to fp16 when generating IR for fp16. - But leave fp32 if node returns shape value even if --data_type=FP16 (look extensions/back/MarkNodesWithShapeValues.py). - """ - enabled = True - force_shape_inference = True - - def run_after(self): - from extensions.back.MarkNodesWithShapeValues import MarkNodesWithShapeValues - return [MarkNodesWithShapeValues] - - def run_before(self): - return [] - - def find_and_replace_pattern(self, graph: Graph): - for node in graph.get_op_nodes(op='Cast'): - if node.dst_type == np.float64: - log.warning('Change data type from {} to {} for node {}'.format(node.dst_type, np.float32, node.name)) - node.dst_type = np.float32 - - ir_data_type = data_type_str_to_np(node.graph.graph['cmd_params'].data_type) - if node.dst_type == np.float32 and ir_data_type == np.float16 and not node.has_and_set('returns_shape_value'): - log.warning('Change data type from {} to {} for node {}'.format(node.dst_type, ir_data_type, node.name)) - node.dst_type = ir_data_type - elif node.has_and_set('returns_shape_value') and node.dst_type == np.float16: - # return back FP32 for all Convert nodes with shape values - log.warning('Change data type from {} to {} for node {} in ShapeOf subgraph'. - format(node.dst_type, np.float32, node.name)) - node.dst_type = np.float32 diff --git a/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py b/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py new file mode 100644 index 00000000000000..b75c7a86c761e4 --- /dev/null +++ b/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py @@ -0,0 +1,100 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import logging as log + +import numpy as np + +from mo.back.replacement import BackReplacementPattern +from mo.graph.graph import Graph +from mo.graph.graph import Node +from mo.middle.passes.convert_data_type import data_type_str_to_np +from mo.utils.error import Error + +operations_with_data_type_attributes = { + 'Cast': {'attr_name': 'dst_type', 'in_ports_to_check': (0,)}, + 'Range': {'attr_name': 'output_type', 'in_ports_to_check': (0, 1, 2)}, +} + + +class ChangeOutputTypeAttributes(BackReplacementPattern): + """ + The transformation changes output type for the specific operations defined in the + operations_with_data_type_attributes dictionary if one of the following conditions is met: + - The operation output type is fp64. Since not all plugins support fp64 data type it is converted to fp32. + - Changes output type from fp32 to fp16 (and ensure that this is possible) when generating fp16 IR. + - Keep operation output type equal to fp32 for operations located in the shape calculation sub-graphs to + avoid floating point overflow. + """ + enabled = True + force_shape_inference = True + + def run_after(self): + from extensions.back.MarkNodesWithShapeValues import MarkNodesWithShapeValues + return [MarkNodesWithShapeValues] + + def run_before(self): + return [] + + def find_and_replace_pattern(self, graph: Graph): + ir_data_type = data_type_str_to_np(graph.graph['cmd_params'].data_type) + + for node in graph.get_op_nodes(): + if node.op in operations_with_data_type_attributes: + dst_type = operations_with_data_type_attributes[node.op]['attr_name'] + node_name = node.soft_get('name', node.id) + assert node.has_valid(dst_type), '{} attribute is missing for node {}'.format(dst_type, node_name) + + final_type = None + if node[dst_type] == np.float64: + final_type = np.float32 + + if node[dst_type] in [np.float32, np.float64] and ir_data_type == np.float16 and \ + not node.has_and_set('returns_shape_value'): + final_type = np.float16 + elif node.has_and_set('returns_shape_value') and node.dst_type == np.float16: + # return back FP32 for all nodes with shape values + final_type = np.float32 + + if final_type is not None: + log.warning('Change data type from {} to {} for node {}'.format(node[dst_type], final_type, + node_name)) + node[dst_type] = final_type + + if final_type == np.float16: + assert_that_is_castable_to_fp16(node) + + +def assert_that_is_castable_to_fp16(node: Node): + op_name = node.soft_get('op') + node_name = node.soft_get('name', node.id) + + for i in operations_with_data_type_attributes[op_name]['in_ports_to_check']: + val = node.in_port(i).data.get_value() + if val is None: + return + + if np.any(val > np.finfo(np.float16).max) or np.any(val < np.finfo(np.float16).min): + raise Error("Try to convert with --data_type=FP32 argument. " + "This model can not be converted to FP16 precision, since " + "'{}' node value {} exceeds FP16 allowed limits: [{}, {}]" + .format(node_name, val, np.finfo(np.float16).min, np.finfo(np.float16).max)) + # further this input values will be rewritten since force_shape_inference=True + node.in_port(i).data.set_value(val.astype(np.float16)) + + original_output = node.out_port(0).data.get_value() + node.infer(node) + casted_output = node.out_port(0).data.get_value() + original_output_len = len(original_output) if hasattr(original_output, '__len__') else None + casted_output_len = len(casted_output) if hasattr(casted_output, '__len__') else None + + if original_output_len != casted_output_len: + raise Error("Try to convert with --data_type=FP32 argument. " + "This model can not be converted to FP16 precision, since " + "after conversion of '{}' node to FP16 output shape {} differs from the original {}." + .format(node_name, casted_output_len, original_output_len)) + + diff_count = np.count_nonzero(np.subtract(original_output, casted_output) > 1.e-4) + if diff_count > 0: + log.warning("{} elements of {} of Range node '{}' output differ from the original values while " + "converting network to FP16 precision".format(diff_count, len(original_output), node_name)) diff --git a/model-optimizer/extensions/back/MarkNodesWithShapeValues.py b/model-optimizer/extensions/back/MarkNodesWithShapeValues.py index 087a7cb4dcfe1c..fa3721bb35c7e4 100644 --- a/model-optimizer/extensions/back/MarkNodesWithShapeValues.py +++ b/model-optimizer/extensions/back/MarkNodesWithShapeValues.py @@ -23,7 +23,6 @@ 'Tile': [1], # repeats input 'TopK': [1], # K input 'Pad': [1, 2], # pads_begin, pads_end - 'Range': [0, 1, 2], # start, stop, step inputs 'OneHot': [1], # depth input } diff --git a/model-optimizer/extensions/ops/range.py b/model-optimizer/extensions/ops/range.py index 78c934105dbef9..4b9e8523766785 100644 --- a/model-optimizer/extensions/ops/range.py +++ b/model-optimizer/extensions/ops/range.py @@ -1,8 +1,6 @@ # Copyright (C) 2018-2021 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import logging as log - import numpy as np from mo.graph.graph import Node, Graph diff --git a/model-optimizer/unit_tests/extensions/back/ChangeOutputTypeAttributes_test.py b/model-optimizer/unit_tests/extensions/back/ChangeOutputTypeAttributes_test.py new file mode 100644 index 00000000000000..b40797397d33f0 --- /dev/null +++ b/model-optimizer/unit_tests/extensions/back/ChangeOutputTypeAttributes_test.py @@ -0,0 +1,122 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import unittest +from copy import deepcopy + +import numpy as np + +from extensions.back.ChangeOutputTypeAttributes import ChangeOutputTypeAttributes +from extensions.ops.Cast import Cast +from extensions.ops.range import Range +from mo.front.common.partial_infer.utils import float32_array +from mo.middle.passes.convert_data_type import convert_blobs, data_type_str_to_np +from mo.middle.passes.infer import partial_infer +from mo.utils.error import Error +from mo.utils.ir_engine.compare_graphs import compare_graphs +from unit_tests.utils.graph import build_graph, result, regular_op_with_empty_data, connect +from unit_tests.utils.graph import valued_const_with_data + + +class ChangeOutputTypeAttributesTests(unittest.TestCase): + + def test_range_correct_case(self): + graph, graph_ref = build_range_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16') + ChangeOutputTypeAttributes().find_and_replace_pattern(graph) + (flag, resp) = compare_graphs(graph, graph_ref, 'res', check_op_attrs=True) + self.assertTrue(flag, resp) + + # starting from ~1000 FP16 absolute difference between neighbor values is more than 1 + # fails because of shape inconsistency + def test_range_different_values(self): + graph, graph_ref = build_range_test_graphs(start=0, limit=50000, delta=1, dst_type_str='FP16') + self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph) + + def test_range_out_of_fp16_max(self): + graph, graph_ref = build_range_test_graphs(start=0, limit=100000, delta=1, dst_type_str='FP16') + self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph) + + def test_range_out_of_fp16_min(self): + graph, graph_ref = build_range_test_graphs(start=0, limit=-100000, delta=-1, dst_type_str='FP16') + self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph) + + def test_cast_correct_case(self): + input_data = np.array([0, 1000, 4, 9, 0]) + graph, graph_ref = build_cast_test_graphs(input_data, dst_type_str='FP16') + ChangeOutputTypeAttributes().find_and_replace_pattern(graph) + (flag, resp) = compare_graphs(graph, graph_ref, 'res', check_op_attrs=True) + self.assertTrue(flag, resp) + + def test_cast_out_of_fp16_max(self): + input_data = np.array([0, 100000, 4, 9, 0]) + graph, graph_ref = build_cast_test_graphs(input_data, dst_type_str='FP16') + self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph) + + def test_cast_out_of_fp16_min(self): + input_data = np.array([0, -100000, 4, 9, 0]) + graph, graph_ref = build_cast_test_graphs(input_data, dst_type_str='FP16') + self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph) + + +def build_range_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16'): + nodes = { + **valued_const_with_data('start', float32_array(start)), + **valued_const_with_data('limit', float32_array(limit)), + **valued_const_with_data('delta', float32_array(delta)), + **regular_op_with_empty_data('range', {'type': 'Range', 'op': 'Range', + 'output_type': np.float32, + 'infer': Range.infer}), + **result('res'), + } + + nodes_ref = deepcopy(nodes) + nodes_ref.update({ + **regular_op_with_empty_data('range', {'type': 'Range', 'op': 'Range', + 'output_type': data_type_str_to_np(dst_type_str), + 'infer': Range.infer}), + }) + + edges = [ + *connect('start', '0:range'), + *connect('limit', '1:range'), + *connect('delta', '2:range'), + *connect('range', 'res'), + ] + graph = build_graph(nodes, edges) + graph_ref = build_graph(nodes_ref, edges) + + graph = partial_infer(graph) + + graph.graph['cmd_params'].data_type = dst_type_str + convert_blobs(graph, dst_type_str) + return graph, graph_ref + + +def build_cast_test_graphs(input_data, dst_type_str='FP16'): + nodes = { + **valued_const_with_data('input', float32_array(input_data)), + **regular_op_with_empty_data('cast', {'type': 'Convert', 'op': 'Cast', + 'dst_type': np.float32, + 'infer': Cast.infer}), + **result('res'), + } + + nodes_ref = deepcopy(nodes) + nodes_ref.update({ + **regular_op_with_empty_data('cast', {'type': 'Convert', 'op': 'Cast', + 'dst_type': data_type_str_to_np(dst_type_str), + 'infer': Cast.infer}), + }) + + edges = [ + *connect('input', 'cast'), + *connect('cast', 'res'), + ] + graph = build_graph(nodes, edges) + graph_ref = build_graph(nodes_ref, edges) + + graph = partial_infer(graph) + + graph.graph['cmd_params'].data_type = dst_type_str + convert_blobs(graph, dst_type_str) + return graph, graph_ref