From d599974b4bc49874be15adb7dc399a7e1abffab0 Mon Sep 17 00:00:00 2001 From: Pavel Esir Date: Fri, 9 Jul 2021 11:15:23 +0300 Subject: [PATCH 1/8] added ChangeRangeOutputType.py --- model-optimizer/automation/package_BOM.txt | 1 + .../extensions/back/ChangeRangeOutputType.py | 71 +++++++++++++++++++ .../back/MarkNodesWithShapeValues.py | 1 - model-optimizer/extensions/ops/range.py | 3 +- .../back/ChangeRangeOutputType_test.py | 69 ++++++++++++++++++ 5 files changed, 142 insertions(+), 3 deletions(-) create mode 100644 model-optimizer/extensions/back/ChangeRangeOutputType.py create mode 100644 model-optimizer/unit_tests/extensions/back/ChangeRangeOutputType_test.py diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index 671c45fd68bd4a..a82eec6181adaf 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -12,6 +12,7 @@ extensions/back/AvgPool.py extensions/back/blob_normalizer.py extensions/back/CellNormalizer.py extensions/back/ChangeCastOutputType.py +extensions/back/ChangeRangeOutputType.py extensions/back/ClampNormalizer.py extensions/back/compress_quantized_weights.py extensions/back/ConvolutionNormalizer.py diff --git a/model-optimizer/extensions/back/ChangeRangeOutputType.py b/model-optimizer/extensions/back/ChangeRangeOutputType.py new file mode 100644 index 00000000000000..1339d946901427 --- /dev/null +++ b/model-optimizer/extensions/back/ChangeRangeOutputType.py @@ -0,0 +1,71 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import logging as log + +import numpy as np + +from mo.back.replacement import BackReplacementPattern +from mo.graph.graph import Graph, Node +from mo.middle.passes.convert_data_type import data_type_str_to_np +from mo.utils.error import Error + + +class ChangeRangeOutputType(BackReplacementPattern): + """ + Change Range 'output_type' from fp64 to fp32 since not all plugins support fp64 data type. + And from fp32 to fp16 when generating IR for fp16. Before changing precision to FP16 ensures that it's possible + """ + enabled = True + force_shape_inference = True + + def run_after(self): + from extensions.back.MarkNodesWithShapeValues import MarkNodesWithShapeValues + return [MarkNodesWithShapeValues] + + def run_before(self): + return [] + + def find_and_replace_pattern(self, graph: Graph): + ir_data_type = data_type_str_to_np(graph.graph['cmd_params'].data_type) + + for node in graph.get_op_nodes(op='Range'): + node_name = node.soft_get('name', node.id) + assert node.has_valid('output_type') + + final_type = None + if node.output_type == np.float64: + final_type = np.float32 + if node.output_type in [np.float32, np.float64] and ir_data_type == np.float16: + final_type = np.float16 + + if final_type == np.float16: + assert_that_is_castable_to_fp16(node) + + if final_type is not None: + node.output_type = final_type + log.warning('Change output_type from {} to {} for Range node {}'.format( + node.output_type, final_type, node_name)) + + +def assert_that_is_castable_to_fp16(node: Node): + node_name = node.soft_get('name', node.id) + start, limit, delta = [node.in_port(i).data.get_value() for i in range(3)] + for val in [start, limit, delta]: + if val > np.finfo(np.float16).max: + raise Error("This model can not be converted to FP16 precision, since " + "Range node '{}' input values {} exceed FP16 max value {}" + .format(node_name, val, np.finfo(np.float16).max)) + + start, limit, delta = [node.in_port(i).data.get_value().astype(np.float16) for i in range(3)] + casted_output = np.arange(start, limit, delta, dtype=node['output_type']) + original_output = node.out_port(0).data.get_value() + if len(original_output) != len(casted_output): + raise Error("This model can not be converted to FP16 precision, since " + "after changing Range node '{}' dtype to FP16 output shape {} differs from original {}" + .format(node_name, len(casted_output), len(original_output))) + + diff_count = np.count_nonzero(np.subtract(original_output, casted_output) > 1.e-4) + if diff_count > 0: + log.warning("{} elements of {} of Range node '{}' output differ from the original values while " + "converting network to FP16 precision".format(diff_count, len(original_output), node_name)) diff --git a/model-optimizer/extensions/back/MarkNodesWithShapeValues.py b/model-optimizer/extensions/back/MarkNodesWithShapeValues.py index 087a7cb4dcfe1c..fa3721bb35c7e4 100644 --- a/model-optimizer/extensions/back/MarkNodesWithShapeValues.py +++ b/model-optimizer/extensions/back/MarkNodesWithShapeValues.py @@ -23,7 +23,6 @@ 'Tile': [1], # repeats input 'TopK': [1], # K input 'Pad': [1, 2], # pads_begin, pads_end - 'Range': [0, 1, 2], # start, stop, step inputs 'OneHot': [1], # depth input } diff --git a/model-optimizer/extensions/ops/range.py b/model-optimizer/extensions/ops/range.py index 78c934105dbef9..66f9be4543e48d 100644 --- a/model-optimizer/extensions/ops/range.py +++ b/model-optimizer/extensions/ops/range.py @@ -1,11 +1,10 @@ # Copyright (C) 2018-2021 Intel Corporation # SPDX-License-Identifier: Apache-2.0 -import logging as log - import numpy as np from mo.graph.graph import Node, Graph +from mo.middle.passes.convert_data_type import data_type_str_to_np from mo.middle.passes.convert_data_type import np_data_type_to_destination_type from mo.ops.op import Op from mo.utils.error import Error diff --git a/model-optimizer/unit_tests/extensions/back/ChangeRangeOutputType_test.py b/model-optimizer/unit_tests/extensions/back/ChangeRangeOutputType_test.py new file mode 100644 index 00000000000000..529720b7943e20 --- /dev/null +++ b/model-optimizer/unit_tests/extensions/back/ChangeRangeOutputType_test.py @@ -0,0 +1,69 @@ +# Copyright (C) 2018-2021 Intel Corporation +# SPDX-License-Identifier: Apache-2.0 + +import unittest +from copy import deepcopy + +import numpy as np + +from extensions.back.ChangeRangeOutputType import ChangeRangeOutputType +from extensions.ops.range import Range +from mo.front.common.partial_infer.utils import float32_array +from mo.middle.passes.convert_data_type import convert_blobs, data_type_str_to_np +from mo.middle.passes.infer import partial_infer +from mo.utils.ir_engine.compare_graphs import compare_graphs +from unit_tests.utils.graph import build_graph, result, regular_op_with_empty_data, connect +from unit_tests.utils.graph import valued_const_with_data + + +class AttributedClampNormalizerTests(unittest.TestCase): + + def test_correct_case(self): + graph, graph_ref = build_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16') + ChangeRangeOutputType().find_and_replace_pattern(graph) + (flag, resp) = compare_graphs(graph, graph_ref, 'res', check_op_attrs=True) + self.assertTrue(flag, resp) + + # starting from ~1000 FP16 absolute difference between neighbor values is more than 1 + # fails because of shape inconsistency + def test_range_different_values(self): + graph, graph_ref = build_test_graphs(start=0, limit=50000, delta=1, dst_type_str='FP16') + self.assertRaises(Exception, ChangeRangeOutputType().find_and_replace_pattern, graph) + + def test_range_out_of_fp16_max(self): + graph, graph_ref = build_test_graphs(start=0, limit=100000, delta=1, dst_type_str='FP16') + self.assertRaises(Exception, ChangeRangeOutputType().find_and_replace_pattern, graph) + + +def build_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16'): + nodes = { + **valued_const_with_data('start', float32_array(start)), + **valued_const_with_data('limit', float32_array(limit)), + **valued_const_with_data('delta', float32_array(delta)), + **regular_op_with_empty_data('range', {'type': 'Range', 'op': 'Range', + 'output_type': np.float32, + 'infer': Range.infer}), + **result('res'), + } + + nodes_ref = deepcopy(nodes) + nodes_ref.update({ + **regular_op_with_empty_data('range', {'type': 'Range', 'op': 'Range', + 'output_type': data_type_str_to_np(dst_type_str), + 'infer': Range.infer}), + }) + + edges = [ + *connect('start', '0:range'), + *connect('limit', '1:range'), + *connect('delta', '2:range'), + *connect('range', 'res'), + ] + graph = build_graph(nodes, edges) + graph_ref = build_graph(nodes_ref, edges) + + graph = partial_infer(graph) + + graph.graph['cmd_params'].data_type = dst_type_str + convert_blobs(graph, dst_type_str) + return graph, graph_ref From 06c6a43595f4f85bc1ca6c8e934fd8d361b4bba1 Mon Sep 17 00:00:00 2001 From: Pavel Esir Date: Mon, 19 Jul 2021 14:55:41 +0300 Subject: [PATCH 2/8] applied review comments --- model-optimizer/extensions/back/ChangeRangeOutputType.py | 6 +++--- model-optimizer/extensions/ops/range.py | 1 - .../extensions/back/ChangeRangeOutputType_test.py | 6 +++++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/model-optimizer/extensions/back/ChangeRangeOutputType.py b/model-optimizer/extensions/back/ChangeRangeOutputType.py index 1339d946901427..0c8c83b27499ca 100644 --- a/model-optimizer/extensions/back/ChangeRangeOutputType.py +++ b/model-optimizer/extensions/back/ChangeRangeOutputType.py @@ -52,10 +52,10 @@ def assert_that_is_castable_to_fp16(node: Node): node_name = node.soft_get('name', node.id) start, limit, delta = [node.in_port(i).data.get_value() for i in range(3)] for val in [start, limit, delta]: - if val > np.finfo(np.float16).max: + if val > np.finfo(np.float16).max or val < np.finfo(np.float16).min: raise Error("This model can not be converted to FP16 precision, since " - "Range node '{}' input values {} exceed FP16 max value {}" - .format(node_name, val, np.finfo(np.float16).max)) + "Range node '{}' input value {} exceeds FP16 allowed limits: [{}, {}]" + .format(node_name, val, np.finfo(np.float16).min, np.finfo(np.float16).max)) start, limit, delta = [node.in_port(i).data.get_value().astype(np.float16) for i in range(3)] casted_output = np.arange(start, limit, delta, dtype=node['output_type']) diff --git a/model-optimizer/extensions/ops/range.py b/model-optimizer/extensions/ops/range.py index 66f9be4543e48d..4b9e8523766785 100644 --- a/model-optimizer/extensions/ops/range.py +++ b/model-optimizer/extensions/ops/range.py @@ -4,7 +4,6 @@ import numpy as np from mo.graph.graph import Node, Graph -from mo.middle.passes.convert_data_type import data_type_str_to_np from mo.middle.passes.convert_data_type import np_data_type_to_destination_type from mo.ops.op import Op from mo.utils.error import Error diff --git a/model-optimizer/unit_tests/extensions/back/ChangeRangeOutputType_test.py b/model-optimizer/unit_tests/extensions/back/ChangeRangeOutputType_test.py index 529720b7943e20..0dd5637c4451cd 100644 --- a/model-optimizer/unit_tests/extensions/back/ChangeRangeOutputType_test.py +++ b/model-optimizer/unit_tests/extensions/back/ChangeRangeOutputType_test.py @@ -16,7 +16,7 @@ from unit_tests.utils.graph import valued_const_with_data -class AttributedClampNormalizerTests(unittest.TestCase): +class ChangeRangeOutputTypeTests(unittest.TestCase): def test_correct_case(self): graph, graph_ref = build_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16') @@ -34,6 +34,10 @@ def test_range_out_of_fp16_max(self): graph, graph_ref = build_test_graphs(start=0, limit=100000, delta=1, dst_type_str='FP16') self.assertRaises(Exception, ChangeRangeOutputType().find_and_replace_pattern, graph) + def test_range_out_of_fp16_min(self): + graph, graph_ref = build_test_graphs(start=0, limit=-100000, delta=-1, dst_type_str='FP16') + self.assertRaises(Exception, ChangeRangeOutputType().find_and_replace_pattern, graph) + def build_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16'): nodes = { From 5310acde2c75325c6a84e4ca64c92260a81c99e1 Mon Sep 17 00:00:00 2001 From: Pavel Esir Date: Tue, 20 Jul 2021 14:12:48 +0300 Subject: [PATCH 3/8] corrected error message - warn user to use FP32 --- model-optimizer/extensions/back/ChangeRangeOutputType.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/model-optimizer/extensions/back/ChangeRangeOutputType.py b/model-optimizer/extensions/back/ChangeRangeOutputType.py index 0c8c83b27499ca..377af99903cd55 100644 --- a/model-optimizer/extensions/back/ChangeRangeOutputType.py +++ b/model-optimizer/extensions/back/ChangeRangeOutputType.py @@ -53,7 +53,8 @@ def assert_that_is_castable_to_fp16(node: Node): start, limit, delta = [node.in_port(i).data.get_value() for i in range(3)] for val in [start, limit, delta]: if val > np.finfo(np.float16).max or val < np.finfo(np.float16).min: - raise Error("This model can not be converted to FP16 precision, since " + raise Error("Try to convert with --data_type=FP32 argument." + "This model can not be converted to FP16 precision, since " "Range node '{}' input value {} exceeds FP16 allowed limits: [{}, {}]" .format(node_name, val, np.finfo(np.float16).min, np.finfo(np.float16).max)) @@ -61,8 +62,9 @@ def assert_that_is_castable_to_fp16(node: Node): casted_output = np.arange(start, limit, delta, dtype=node['output_type']) original_output = node.out_port(0).data.get_value() if len(original_output) != len(casted_output): - raise Error("This model can not be converted to FP16 precision, since " - "after changing Range node '{}' dtype to FP16 output shape {} differs from original {}" + raise Error("Try to convert with --data_type=FP32 argument." + "This model can not be converted to FP16 precision, since " + "after changing Range node '{}' dtype to FP16 output shape {} differs from original {}." .format(node_name, len(casted_output), len(original_output))) diff_count = np.count_nonzero(np.subtract(original_output, casted_output) > 1.e-4) From 423a6b8030d9e96599064c7c5d00d1b88decc13a Mon Sep 17 00:00:00 2001 From: Pavel Esir Date: Thu, 22 Jul 2021 11:03:03 +0300 Subject: [PATCH 4/8] renamed ChangeCastOutputType.py et ell. --- model-optimizer/automation/package_BOM.txt | 3 +- ...tType.py => ChangeOutputTypeAttributes.py} | 2 +- .../extensions/back/ChangeRangeOutputType.py | 73 ------------------- ....py => ChangeOutputTypeAttributes_test.py} | 0 4 files changed, 2 insertions(+), 76 deletions(-) rename model-optimizer/extensions/back/{ChangeCastOutputType.py => ChangeOutputTypeAttributes.py} (97%) delete mode 100644 model-optimizer/extensions/back/ChangeRangeOutputType.py rename model-optimizer/unit_tests/extensions/back/{ChangeRangeOutputType_test.py => ChangeOutputTypeAttributes_test.py} (100%) diff --git a/model-optimizer/automation/package_BOM.txt b/model-optimizer/automation/package_BOM.txt index 0d60150b02c8c6..e30464354aaed4 100644 --- a/model-optimizer/automation/package_BOM.txt +++ b/model-optimizer/automation/package_BOM.txt @@ -11,8 +11,7 @@ extensions/back/__init__.py extensions/back/AvgPool.py extensions/back/blob_normalizer.py extensions/back/CellNormalizer.py -extensions/back/ChangeCastOutputType.py -extensions/back/ChangeRangeOutputType.py +extensions/back/ChangeOutputTypeAttributes.py extensions/back/ClampNormalizer.py extensions/back/compress_quantized_weights.py extensions/back/ConvolutionNormalizer.py diff --git a/model-optimizer/extensions/back/ChangeCastOutputType.py b/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py similarity index 97% rename from model-optimizer/extensions/back/ChangeCastOutputType.py rename to model-optimizer/extensions/back/ChangeOutputTypeAttributes.py index 976b6b50a29136..0a8141d8858cbe 100644 --- a/model-optimizer/extensions/back/ChangeCastOutputType.py +++ b/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py @@ -10,7 +10,7 @@ from mo.middle.passes.convert_data_type import data_type_str_to_np -class ChangeCastOutputType(BackReplacementPattern): +class ChangeOutputTypeAttributes(BackReplacementPattern): """ Change the Cast dst_type from fp64 to fp32 since not all plugins support fp64 data type. Change the Cast dst_type from fp32 to fp16 when generating IR for fp16. diff --git a/model-optimizer/extensions/back/ChangeRangeOutputType.py b/model-optimizer/extensions/back/ChangeRangeOutputType.py deleted file mode 100644 index 377af99903cd55..00000000000000 --- a/model-optimizer/extensions/back/ChangeRangeOutputType.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (C) 2018-2021 Intel Corporation -# SPDX-License-Identifier: Apache-2.0 - -import logging as log - -import numpy as np - -from mo.back.replacement import BackReplacementPattern -from mo.graph.graph import Graph, Node -from mo.middle.passes.convert_data_type import data_type_str_to_np -from mo.utils.error import Error - - -class ChangeRangeOutputType(BackReplacementPattern): - """ - Change Range 'output_type' from fp64 to fp32 since not all plugins support fp64 data type. - And from fp32 to fp16 when generating IR for fp16. Before changing precision to FP16 ensures that it's possible - """ - enabled = True - force_shape_inference = True - - def run_after(self): - from extensions.back.MarkNodesWithShapeValues import MarkNodesWithShapeValues - return [MarkNodesWithShapeValues] - - def run_before(self): - return [] - - def find_and_replace_pattern(self, graph: Graph): - ir_data_type = data_type_str_to_np(graph.graph['cmd_params'].data_type) - - for node in graph.get_op_nodes(op='Range'): - node_name = node.soft_get('name', node.id) - assert node.has_valid('output_type') - - final_type = None - if node.output_type == np.float64: - final_type = np.float32 - if node.output_type in [np.float32, np.float64] and ir_data_type == np.float16: - final_type = np.float16 - - if final_type == np.float16: - assert_that_is_castable_to_fp16(node) - - if final_type is not None: - node.output_type = final_type - log.warning('Change output_type from {} to {} for Range node {}'.format( - node.output_type, final_type, node_name)) - - -def assert_that_is_castable_to_fp16(node: Node): - node_name = node.soft_get('name', node.id) - start, limit, delta = [node.in_port(i).data.get_value() for i in range(3)] - for val in [start, limit, delta]: - if val > np.finfo(np.float16).max or val < np.finfo(np.float16).min: - raise Error("Try to convert with --data_type=FP32 argument." - "This model can not be converted to FP16 precision, since " - "Range node '{}' input value {} exceeds FP16 allowed limits: [{}, {}]" - .format(node_name, val, np.finfo(np.float16).min, np.finfo(np.float16).max)) - - start, limit, delta = [node.in_port(i).data.get_value().astype(np.float16) for i in range(3)] - casted_output = np.arange(start, limit, delta, dtype=node['output_type']) - original_output = node.out_port(0).data.get_value() - if len(original_output) != len(casted_output): - raise Error("Try to convert with --data_type=FP32 argument." - "This model can not be converted to FP16 precision, since " - "after changing Range node '{}' dtype to FP16 output shape {} differs from original {}." - .format(node_name, len(casted_output), len(original_output))) - - diff_count = np.count_nonzero(np.subtract(original_output, casted_output) > 1.e-4) - if diff_count > 0: - log.warning("{} elements of {} of Range node '{}' output differ from the original values while " - "converting network to FP16 precision".format(diff_count, len(original_output), node_name)) diff --git a/model-optimizer/unit_tests/extensions/back/ChangeRangeOutputType_test.py b/model-optimizer/unit_tests/extensions/back/ChangeOutputTypeAttributes_test.py similarity index 100% rename from model-optimizer/unit_tests/extensions/back/ChangeRangeOutputType_test.py rename to model-optimizer/unit_tests/extensions/back/ChangeOutputTypeAttributes_test.py From d9090eea27bf17142fb49b5e2bc2e960d72de2c4 Mon Sep 17 00:00:00 2001 From: Pavel Esir Date: Thu, 22 Jul 2021 11:04:06 +0300 Subject: [PATCH 5/8] merged ChangeRangeOutputType.py, ChangeCastOutputType.py into a singe file --- .../back/ChangeOutputTypeAttributes.py | 85 +++++++++++++++---- .../back/ChangeOutputTypeAttributes_test.py | 73 +++++++++++++--- 2 files changed, 129 insertions(+), 29 deletions(-) diff --git a/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py b/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py index 0a8141d8858cbe..0870c43532c8d1 100644 --- a/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py +++ b/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py @@ -7,14 +7,22 @@ from mo.back.replacement import BackReplacementPattern from mo.graph.graph import Graph +from mo.graph.graph import Node from mo.middle.passes.convert_data_type import data_type_str_to_np +from mo.utils.error import Error + +operations_with_data_type_attributes = { + 'Cast': {'attr_name': 'dst_type', 'in_ports_to_check': (0,)}, + 'Range': {'attr_name': 'output_type', 'in_ports_to_check': (0, 1, 2)}, +} class ChangeOutputTypeAttributes(BackReplacementPattern): """ - Change the Cast dst_type from fp64 to fp32 since not all plugins support fp64 data type. - Change the Cast dst_type from fp32 to fp16 when generating IR for fp16. - But leave fp32 if node returns shape value even if --data_type=FP16 (look extensions/back/MarkNodesWithShapeValues.py). + For operations that allow specify destination type by attribute this transformation + changes destination type from fp64 to fp32 since not all plugins support fp64 data type. + Changes destination type from fp32 to fp16 (and ensure that this is possible) when generating IR for fp16. + But leave fp32 if node returns shape value even if --data_type=FP16 (look extensions/back/MarkNodesWithShapeValues.py) """ enabled = True force_shape_inference = True @@ -27,17 +35,60 @@ def run_before(self): return [] def find_and_replace_pattern(self, graph: Graph): - for node in graph.get_op_nodes(op='Cast'): - if node.dst_type == np.float64: - log.warning('Change data type from {} to {} for node {}'.format(node.dst_type, np.float32, node.name)) - node.dst_type = np.float32 - - ir_data_type = data_type_str_to_np(node.graph.graph['cmd_params'].data_type) - if node.dst_type == np.float32 and ir_data_type == np.float16 and not node.has_and_set('returns_shape_value'): - log.warning('Change data type from {} to {} for node {}'.format(node.dst_type, ir_data_type, node.name)) - node.dst_type = ir_data_type - elif node.has_and_set('returns_shape_value') and node.dst_type == np.float16: - # return back FP32 for all Convert nodes with shape values - log.warning('Change data type from {} to {} for node {} in ShapeOf subgraph'. - format(node.dst_type, np.float32, node.name)) - node.dst_type = np.float32 + ir_data_type = data_type_str_to_np(graph.graph['cmd_params'].data_type) + + for op_name, op_map in operations_with_data_type_attributes.items(): + dst_type = op_map['attr_name'] + for node in graph.get_op_nodes(op=op_name): + node_name = node.soft_get('name', node.id) + assert node.has_valid(dst_type) + + final_type = None + if node[dst_type] == np.float64: + final_type = np.float32 + + if node[dst_type] in [np.float32, np.float64] and ir_data_type == np.float16 and \ + not node.has_and_set('returns_shape_value'): + final_type = np.float16 + elif node.has_and_set('returns_shape_value') and node.dst_type == np.float16: + # return back FP32 for all nodes with shape values + final_type = np.float32 + + if final_type is not None: + node[dst_type] = final_type + log.warning('Change data type from {} to {} for node {}'.format(node[dst_type], final_type, + node_name)) + + if final_type == np.float16: + assert_that_is_castable_to_fp16(node) + + +def assert_that_is_castable_to_fp16(node: Node): + node_type = node.soft_get('type') + node_name = node.soft_get('name', node.id) + + for i in operations_with_data_type_attributes[node_type]['in_ports_to_check']: + val = node.in_port(i).data.get_value() + + if np.any(val > np.finfo(np.float16).max) or np.any(val < np.finfo(np.float16).min): + raise Error("Try to convert with --data_type=FP32 argument. " + "This model can not be converted to FP16 precision, since " + "'{}' node value {} exceeds FP16 allowed limits: [{}, {}]" + .format(node_name, val, np.finfo(np.float16).min, np.finfo(np.float16).max)) + # further this input values will be rewritten since force_shape_inference=True + node.in_port(i).data.set_value(val.astype(np.float16)) + + original_output = node.out_port(0).data.get_value() + node.infer(node) + casted_output = node.out_port(0).data.get_value() + + if len(original_output) != len(casted_output): + raise Error("Try to convert with --data_type=FP32 argument. " + "This model can not be converted to FP16 precision, since " + "after '{}' node dtype to FP16 output shape {} differs from original {}." + .format(node_name, len(casted_output), len(original_output))) + + diff_count = np.count_nonzero(np.subtract(original_output, casted_output) > 1.e-4) + if diff_count > 0: + log.warning("{} elements of {} of Range node '{}' output differ from the original values while " + "converting network to FP16 precision".format(diff_count, len(original_output), node_name)) diff --git a/model-optimizer/unit_tests/extensions/back/ChangeOutputTypeAttributes_test.py b/model-optimizer/unit_tests/extensions/back/ChangeOutputTypeAttributes_test.py index 0dd5637c4451cd..f8fb13819308c8 100644 --- a/model-optimizer/unit_tests/extensions/back/ChangeOutputTypeAttributes_test.py +++ b/model-optimizer/unit_tests/extensions/back/ChangeOutputTypeAttributes_test.py @@ -6,40 +6,59 @@ import numpy as np -from extensions.back.ChangeRangeOutputType import ChangeRangeOutputType +from extensions.back.ChangeOutputTypeAttributes import ChangeOutputTypeAttributes +from extensions.ops.Cast import Cast from extensions.ops.range import Range from mo.front.common.partial_infer.utils import float32_array from mo.middle.passes.convert_data_type import convert_blobs, data_type_str_to_np from mo.middle.passes.infer import partial_infer +from mo.utils.error import Error from mo.utils.ir_engine.compare_graphs import compare_graphs from unit_tests.utils.graph import build_graph, result, regular_op_with_empty_data, connect from unit_tests.utils.graph import valued_const_with_data -class ChangeRangeOutputTypeTests(unittest.TestCase): +class ChangeOutputTypeAttributesTests(unittest.TestCase): - def test_correct_case(self): - graph, graph_ref = build_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16') - ChangeRangeOutputType().find_and_replace_pattern(graph) + def test_range_correct_case(self): + graph, graph_ref = build_range_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16') + ChangeOutputTypeAttributes().find_and_replace_pattern(graph) (flag, resp) = compare_graphs(graph, graph_ref, 'res', check_op_attrs=True) self.assertTrue(flag, resp) # starting from ~1000 FP16 absolute difference between neighbor values is more than 1 # fails because of shape inconsistency def test_range_different_values(self): - graph, graph_ref = build_test_graphs(start=0, limit=50000, delta=1, dst_type_str='FP16') - self.assertRaises(Exception, ChangeRangeOutputType().find_and_replace_pattern, graph) + graph, graph_ref = build_range_test_graphs(start=0, limit=50000, delta=1, dst_type_str='FP16') + self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph) def test_range_out_of_fp16_max(self): - graph, graph_ref = build_test_graphs(start=0, limit=100000, delta=1, dst_type_str='FP16') - self.assertRaises(Exception, ChangeRangeOutputType().find_and_replace_pattern, graph) + graph, graph_ref = build_range_test_graphs(start=0, limit=100000, delta=1, dst_type_str='FP16') + self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph) def test_range_out_of_fp16_min(self): - graph, graph_ref = build_test_graphs(start=0, limit=-100000, delta=-1, dst_type_str='FP16') - self.assertRaises(Exception, ChangeRangeOutputType().find_and_replace_pattern, graph) + graph, graph_ref = build_range_test_graphs(start=0, limit=-100000, delta=-1, dst_type_str='FP16') + self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph) + def test_cast_correct_case(self): + input_data = np.array([0, 1000, 4, 9, 0]) + graph, graph_ref = build_cast_test_graphs(input_data, dst_type_str='FP16') + ChangeOutputTypeAttributes().find_and_replace_pattern(graph) + (flag, resp) = compare_graphs(graph, graph_ref, 'res', check_op_attrs=True) + self.assertTrue(flag, resp) + + def test_cast_out_of_fp16_max(self): + input_data = np.array([0, 100000, 4, 9, 0]) + graph, graph_ref = build_cast_test_graphs(input_data, dst_type_str='FP16') + self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph) + + def test_cast_out_of_fp16_min(self): + input_data = np.array([0, -100000, 4, 9, 0]) + graph, graph_ref = build_cast_test_graphs(input_data, dst_type_str='FP16') + self.assertRaises(Error, ChangeOutputTypeAttributes().find_and_replace_pattern, graph) -def build_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16'): + +def build_range_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16'): nodes = { **valued_const_with_data('start', float32_array(start)), **valued_const_with_data('limit', float32_array(limit)), @@ -71,3 +90,33 @@ def build_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16'): graph.graph['cmd_params'].data_type = dst_type_str convert_blobs(graph, dst_type_str) return graph, graph_ref + + +def build_cast_test_graphs(input_data, dst_type_str='FP16'): + nodes = { + **valued_const_with_data('input', float32_array(input_data)), + **regular_op_with_empty_data('cast', {'type': 'Cast', 'op': 'Cast', + 'dst_type': np.float32, + 'infer': Cast.infer}), + **result('res'), + } + + nodes_ref = deepcopy(nodes) + nodes_ref.update({ + **regular_op_with_empty_data('cast', {'type': 'Cast', 'op': 'Cast', + 'dst_type': data_type_str_to_np(dst_type_str), + 'infer': Cast.infer}), + }) + + edges = [ + *connect('input', 'cast'), + *connect('cast', 'res'), + ] + graph = build_graph(nodes, edges) + graph_ref = build_graph(nodes_ref, edges) + + graph = partial_infer(graph) + + graph.graph['cmd_params'].data_type = dst_type_str + convert_blobs(graph, dst_type_str) + return graph, graph_ref From aba7a6467255b86c8ddc63252babe67147f527d3 Mon Sep 17 00:00:00 2001 From: Pavel Esir Date: Thu, 22 Jul 2021 11:53:33 +0300 Subject: [PATCH 6/8] corrections --- .../back/ChangeOutputTypeAttributes.py | 18 +++++++++++------- .../back/ChangeOutputTypeAttributes_test.py | 4 ++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py b/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py index 0870c43532c8d1..4806dd13003990 100644 --- a/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py +++ b/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py @@ -19,9 +19,9 @@ class ChangeOutputTypeAttributes(BackReplacementPattern): """ - For operations that allow specify destination type by attribute this transformation - changes destination type from fp64 to fp32 since not all plugins support fp64 data type. - Changes destination type from fp32 to fp16 (and ensure that this is possible) when generating IR for fp16. + For operations that allow specify output type by attribute this transformation + changes output type from fp64 to fp32 since not all plugins support fp64 data type. + Changes output type from fp32 to fp16 (and ensure that this is possible) when generating IR for fp16. But leave fp32 if node returns shape value even if --data_type=FP16 (look extensions/back/MarkNodesWithShapeValues.py) """ enabled = True @@ -64,11 +64,13 @@ def find_and_replace_pattern(self, graph: Graph): def assert_that_is_castable_to_fp16(node: Node): - node_type = node.soft_get('type') + op_name = node.soft_get('op') node_name = node.soft_get('name', node.id) - for i in operations_with_data_type_attributes[node_type]['in_ports_to_check']: + for i in operations_with_data_type_attributes[op_name]['in_ports_to_check']: val = node.in_port(i).data.get_value() + if val is None: + return if np.any(val > np.finfo(np.float16).max) or np.any(val < np.finfo(np.float16).min): raise Error("Try to convert with --data_type=FP32 argument. " @@ -81,12 +83,14 @@ def assert_that_is_castable_to_fp16(node: Node): original_output = node.out_port(0).data.get_value() node.infer(node) casted_output = node.out_port(0).data.get_value() + original_output_len = len(original_output) if hasattr(original_output, '__len__') else None + casted_output_len = len(casted_output) if hasattr(casted_output, '__len__') else None - if len(original_output) != len(casted_output): + if original_output_len != casted_output_len: raise Error("Try to convert with --data_type=FP32 argument. " "This model can not be converted to FP16 precision, since " "after '{}' node dtype to FP16 output shape {} differs from original {}." - .format(node_name, len(casted_output), len(original_output))) + .format(node_name, casted_output_len, original_output_len)) diff_count = np.count_nonzero(np.subtract(original_output, casted_output) > 1.e-4) if diff_count > 0: diff --git a/model-optimizer/unit_tests/extensions/back/ChangeOutputTypeAttributes_test.py b/model-optimizer/unit_tests/extensions/back/ChangeOutputTypeAttributes_test.py index f8fb13819308c8..b40797397d33f0 100644 --- a/model-optimizer/unit_tests/extensions/back/ChangeOutputTypeAttributes_test.py +++ b/model-optimizer/unit_tests/extensions/back/ChangeOutputTypeAttributes_test.py @@ -95,7 +95,7 @@ def build_range_test_graphs(start=0, limit=10, delta=1, dst_type_str='FP16'): def build_cast_test_graphs(input_data, dst_type_str='FP16'): nodes = { **valued_const_with_data('input', float32_array(input_data)), - **regular_op_with_empty_data('cast', {'type': 'Cast', 'op': 'Cast', + **regular_op_with_empty_data('cast', {'type': 'Convert', 'op': 'Cast', 'dst_type': np.float32, 'infer': Cast.infer}), **result('res'), @@ -103,7 +103,7 @@ def build_cast_test_graphs(input_data, dst_type_str='FP16'): nodes_ref = deepcopy(nodes) nodes_ref.update({ - **regular_op_with_empty_data('cast', {'type': 'Cast', 'op': 'Cast', + **regular_op_with_empty_data('cast', {'type': 'Convert', 'op': 'Cast', 'dst_type': data_type_str_to_np(dst_type_str), 'infer': Cast.infer}), }) From 6e14a506e0267e004763e4585e9f7e32a21f9293 Mon Sep 17 00:00:00 2001 From: Pavel Esir Date: Tue, 27 Jul 2021 11:24:36 +0300 Subject: [PATCH 7/8] typo fix --- model-optimizer/extensions/back/ChangeOutputTypeAttributes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py b/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py index 4806dd13003990..fc7e84e5c7b3ef 100644 --- a/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py +++ b/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py @@ -55,9 +55,9 @@ def find_and_replace_pattern(self, graph: Graph): final_type = np.float32 if final_type is not None: - node[dst_type] = final_type log.warning('Change data type from {} to {} for node {}'.format(node[dst_type], final_type, node_name)) + node[dst_type] = final_type if final_type == np.float16: assert_that_is_castable_to_fp16(node) From bb73fb124e7f3a1098603b80379c9c422553950a Mon Sep 17 00:00:00 2001 From: Pavel Esir Date: Fri, 30 Jul 2021 10:12:02 +0300 Subject: [PATCH 8/8] applied comments: faster find_and_replace loop, wording correction --- .../back/ChangeOutputTypeAttributes.py | 20 ++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py b/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py index fc7e84e5c7b3ef..b75c7a86c761e4 100644 --- a/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py +++ b/model-optimizer/extensions/back/ChangeOutputTypeAttributes.py @@ -19,10 +19,12 @@ class ChangeOutputTypeAttributes(BackReplacementPattern): """ - For operations that allow specify output type by attribute this transformation - changes output type from fp64 to fp32 since not all plugins support fp64 data type. - Changes output type from fp32 to fp16 (and ensure that this is possible) when generating IR for fp16. - But leave fp32 if node returns shape value even if --data_type=FP16 (look extensions/back/MarkNodesWithShapeValues.py) + The transformation changes output type for the specific operations defined in the + operations_with_data_type_attributes dictionary if one of the following conditions is met: + - The operation output type is fp64. Since not all plugins support fp64 data type it is converted to fp32. + - Changes output type from fp32 to fp16 (and ensure that this is possible) when generating fp16 IR. + - Keep operation output type equal to fp32 for operations located in the shape calculation sub-graphs to + avoid floating point overflow. """ enabled = True force_shape_inference = True @@ -37,11 +39,11 @@ def run_before(self): def find_and_replace_pattern(self, graph: Graph): ir_data_type = data_type_str_to_np(graph.graph['cmd_params'].data_type) - for op_name, op_map in operations_with_data_type_attributes.items(): - dst_type = op_map['attr_name'] - for node in graph.get_op_nodes(op=op_name): + for node in graph.get_op_nodes(): + if node.op in operations_with_data_type_attributes: + dst_type = operations_with_data_type_attributes[node.op]['attr_name'] node_name = node.soft_get('name', node.id) - assert node.has_valid(dst_type) + assert node.has_valid(dst_type), '{} attribute is missing for node {}'.format(dst_type, node_name) final_type = None if node[dst_type] == np.float64: @@ -89,7 +91,7 @@ def assert_that_is_castable_to_fp16(node: Node): if original_output_len != casted_output_len: raise Error("Try to convert with --data_type=FP32 argument. " "This model can not be converted to FP16 precision, since " - "after '{}' node dtype to FP16 output shape {} differs from original {}." + "after conversion of '{}' node to FP16 output shape {} differs from the original {}." .format(node_name, casted_output_len, original_output_len)) diff_count = np.count_nonzero(np.subtract(original_output, casted_output) > 1.e-4)