Merge branch 'feature/20180425-file-format-converter-update-nnb' into…

… 'feature/20180401-file-format-converter'

Feature/20180425 file format converter update nnb

See merge request nnabla/nnabla!167

build-tools/code_generator/generate.py

@@ -14,13 +14,27 @@
 
 import io
 import os
-from os.path import abspath, dirname, join
+from os.path import abspath, dirname, join, exists
 
 here = abspath(dirname(abspath(__file__)))
 base = abspath(join(here, '../..'))
 
 import code_generator_utils as utils
-
+import yaml
+
+def type_to_pack_format(typestring):
+    fmt = None
+    if typestring == 'bool':
+        fmt = 'B'
+    elif typestring == 'double' or typestring == 'float':
+        fmt = 'f'
+    elif typestring == 'int64':
+        fmt = 'i'
+    elif typestring == 'repeated int64' or typestring == 'Shape':
+        fmt = 'iI'
+    elif typestring == 'string':
+        fmt = 'i'
+    return fmt
 
 def generate_cpp_utils(function_info):
     function_list = utils.info_to_list(function_info)
@@ -55,6 +69,53 @@ def generate_solver_python_intereface(solver_info):
     utils.generate_from_template(
         join(base, 'python/src/nnabla/solver.pxd.tmpl'), solver_info=solver_info)
 
+def generate_function_order(function_info):
+    with open(join(base, 'python/src/nnabla/utils/converter/utils.py')) as f:
+        exec(f.read())
+    order_yaml = join(base, 'python/src/nnabla/utils/converter/function_order.yaml')
+    order_info= None
+    if exists(order_yaml):
+        with open(order_yaml, 'r') as f:
+            order_info = utils.load_yaml_ordered(f)
+
+    if order_info is None:
+        order_info= OrderedDict()    
+
+    func_id = 0
+    for func, func_info in function_info.items():
+        fmt = ''
+        if 'arguments' in func_info:
+            fmt = '_'
+            for arg, arg_info in func_info['arguments'].items():
+                fmt += type_to_pack_format(arg_info['type'])
+
+        name = func+fmt
+        if name not in order_info:
+            while func_id in order_info.values():
+                func_id += 1
+            order_info[name] = func_id
+
+        func_id += 1
+    with open(order_yaml, 'w') as f:
+        f.write('# Copyright (c) 2017 Sony Corporation. All Rights Reserved.\n')
+        f.write('#\n')
+        f.write('# Licensed under the Apache License, Version 2.0 (the "License");\n')
+        f.write('# you may not use this file except in compliance with the License.\n')
+        f.write('# You may obtain a copy of the License at\n')
+        f.write('#\n')
+        f.write('#     http://www.apache.org/licenses/LICENSE-2.0\n')
+        f.write('#\n')
+        f.write('# Unless required by applicable law or agreed to in writing, software\n')
+        f.write('# distributed under the License is distributed on an "AS IS" BASIS,\n')
+        f.write('# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n')
+        f.write('# See the License for the specific language governing permissions and\n')
+        f.write('# limitations under the License.\n')
+        f.write('#\n')
+        f.write('# DO NOT EDIT THIS FILE!\n')
+        f.write('# THIS FILE IS GENERATED BY CODE GENERATOR BUT ITS COMMITED INTO\n')
+        f.write('# SOURCE TREE TO MAKE FUNCTION ID PERSIST.\n')
+        f.write('\n')
+        f.write(yaml.dump(order_info, default_flow_style=False))
 
 def generate():
     function_info = utils.load_function_info(flatten=True)
@@ -73,6 +134,7 @@ def generate():
     generate_python_utils(function_info)
     generate_proto(function_info, solver_info)
     generate_cpp_utils(function_info)
+    generate_function_order(function_info)
 
     # Generate function skeltons if new ones are added to functions.yaml and function_types.yaml.
     utils.generate_skelton_function_impl(

docker/dist/Dockerfile

@@ -112,6 +112,6 @@ RUN umask 0 \
     ; pip install -r /tmp/deps/setup_requirements.txt \
     ; pip install -r /tmp/deps/requirements.txt \
     ; pip install -r /tmp/deps/test_requirements.txt \
-    ; pip install pytest-pep8 pytest-cov \
+    ; pip install pytest-pep8 pytest-cov autopep8 \
     ; source deactivate \
     ; done

python/setup.py

@@ -185,7 +185,9 @@ def extopts(library_name, library_dir):
 
     shutil.copyfile(library_path, os.path.join(path_pkg, library_file_name))
     package_data = {"nnabla": [
-        library_file_name, 'nnabla.conf', 'utils/converter/functions.yaml']}
+        library_file_name, 'nnabla.conf',
+                           'utils/converter/functions.yaml',
+                           'utils/converter/function_order.yaml']}
 
     for root, dirs, files in os.walk(os.path.join(build_dir, 'bin')):
         for fn in files:

python/src/nnabla/parameter.py

@@ -27,6 +27,10 @@
 from nnabla.logger import logger
 import nnabla.utils.nnabla_pb2 as nnabla_pb2
 
+# TODO temporary work around to suppress FutureWarning message.
+import warnings
+warnings.simplefilter('ignore', category=FutureWarning)
+
 current_scope = OrderedDict()
 root_scope = current_scope
 

python/src/nnabla/utils/cli/convert.py

@@ -80,4 +80,13 @@ def add_convert_command(subparsers):
                            help='[export][NNP] store parameter into nntxt')
     subparser.add_argument('--nnp-exclude-parameter', action='store_true',
                            help='[export][NNP] output without parameter')
+
+    # Both NNB and CSRC
+    subparser.add_argument('-T', '--default-variable-type', type=str, nargs=1, default=['FLOAT32'],
+                           help='Default type of variable')
+    subparser.add_argument('-G', '--settings-template', type=str, nargs=1, default=None,
+                           help='Get template for settings')
+    subparser.add_argument('-s', '--settings', type=str, nargs=1, default=None,
+                           help='Settings in YAML format file.')
+
     subparser.set_defaults(func=convert_command)

python/src/nnabla/utils/converter/__init__.py

@@ -15,10 +15,12 @@
 from .supported_info import formats
 from .supported_info import extensions
 from .convert_files import convert_files
+from .utils import type_to_pack_format
 from .utils import get_category_info_string
 from .utils import get_category_info_version
 from .utils import get_category_info
 from .utils import get_function_info
+from .utils import load_yaml_ordered
 from .utils import select_executor
 from .utils import search_network
 from .utils import calc_shape_size

python/src/nnabla/utils/converter/convert_files.py

@@ -33,7 +33,8 @@ def export_from_nnp(args, nnp, output):
         NnpExporter(nnp, args.batch_size, parameter_type).export(output)
 
     elif output_ext == '.nnb':
-        NnbExporter(nnp, args.batch_size).export(output)
+        NnbExporter(nnp, args.batch_size).export(
+            output, args.settings_template, args.settings, args.default_variable_type)
 
     elif os.path.isdir(output) and args.export_format == 'CSRC':
         CsrcExporter(nnp, args.batch_size).export(output)

python/src/nnabla/utils/converter/function_order.yaml

@@ -0,0 +1,139 @@
+# Copyright (c) 2017 Sony Corporation. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# DO NOT EDIT THIS FILE!
+# THIS FILE IS GENERATED BY CODE GENERATOR BUT ITS COMMITED INTO
+# SOURCE TREE TO MAKE FUNCTION ID PERSIST.
+
+Affine_i: 0
+Convolution_iiIiIiIi: 1
+DepthwiseConvolution_iiIiIiIi: 2
+Deconvolution_iiIiIiIi: 3
+DepthwiseDeconvolution_iiIiIiIi: 4
+MaxPooling_iIiIBiI: 5
+AveragePooling_iIiIBiIB: 6
+GlobalAveragePooling: 7
+SumPooling_iIiIBiI: 8
+Unpooling_iI: 9
+Embed: 10
+Sigmoid: 11
+Swish: 12
+Tanh: 13
+ReLU_B: 14
+LeakyReLU_f: 15
+Softmax_i: 16
+ELU_f: 17
+SELU_ff: 18
+CReLU_i: 19
+CELU_fi: 20
+PReLU_i: 21
+BatchNormalization_iIffB: 22
+MeanSubtraction_iB: 23
+Sum_iIB: 24
+Mean_iIB: 25
+Max_iIB: 26
+Min_iIB: 27
+Prod_iIB: 28
+ReduceSum: 29
+ReduceMean: 30
+Add2_B: 31
+BcAdd2: 32
+Sub2: 33
+Mul2: 34
+Div2: 35
+Pow2: 36
+AddScalar_f: 37
+MulScalar_f: 38
+PowScalar_f: 39
+RSubScalar_f: 40
+RDivScalar_f: 41
+RPowScalar_f: 42
+Sign_f: 43
+Minimum2: 44
+Maximum2: 45
+MinimumScalar_f: 46
+MaximumScalar_f: 47
+LogicalAnd: 48
+LogicalOr: 49
+LogicalXor: 50
+Equal: 51
+NotEqual: 52
+GreaterEqual: 53
+Greater: 54
+LessEqual: 55
+Less: 56
+LogicalAndScalar_B: 57
+LogicalOrScalar_B: 58
+LogicalXorScalar_B: 59
+EqualScalar_f: 60
+NotEqualScalar_f: 61
+GreaterEqualScalar_f: 62
+GreaterScalar_f: 63
+LessEqualScalar_f: 64
+LessScalar_f: 65
+LogicalNot: 66
+Constant_fiI: 67
+Abs: 68
+Exp: 69
+Log: 70
+Identity: 71
+BatchMatmul_BB: 72
+Round_: 73
+Concatenate_i: 74
+Split_i: 75
+Stack_i: 76
+Slice_iIiIiI: 77
+Transpose_iI: 78
+Broadcast_iI: 79
+OneHot_iI: 80
+Flip_iI: 81
+Shift_iIi: 82
+Reshape_iI: 83
+MatrixDiag: 84
+MatrixDiagPart: 85
+Dropout_fi: 86
+TopKData_iBBi: 87
+TopKGrad_iBi: 88
+Rand_ffiIi: 89
+Randint_iiiIi: 90
+Randn_ffiIi: 91
+RandomCrop_iIii: 92
+RandomFlip_iIii: 93
+RandomShift_iIiii: 94
+ImageAugmentation_iIiIfffffBBfBffBfi: 95
+SigmoidCrossEntropy: 96
+BinaryCrossEntropy: 97
+SoftmaxCrossEntropy_i: 98
+CategoricalCrossEntropy_i: 99
+SquaredError: 100
+AbsoluteError: 101
+HuberLoss_f: 102
+EpsilonInsensitiveLoss_f: 103
+KLMultinomial_i: 104
+BinarySigmoid: 105
+BinaryTanh: 106
+BinaryConnectAffine_i: 107
+BinaryConnectConvolution_iiIiIiIi: 108
+BinaryWeightAffine_i: 109
+BinaryWeightConvolution_iiIiIiIi: 110
+INQAffine_iiiIii: 111
+INQConvolution_iiIiIiIiiiIii: 112
+FixedPointQuantize_BifB: 113
+Pow2Quantize_BBiiB: 114
+TopNError_ii: 115
+BinaryError: 116
+ConfusionMatrix_i: 117
+VATNoise_if: 118
+Unlink: 119
+Sink_B: 120

python/src/nnabla/utils/converter/nnabla/exporter.py

@@ -19,6 +19,10 @@
 import tempfile
 import zipfile
 
+# TODO temporary work around to suppress FutureWarning message.
+import warnings
+warnings.simplefilter('ignore', category=FutureWarning)
+
 from nnabla.utils import nnabla_pb2
 
 

python/src/nnabla/utils/converter/nnabla/reader.py

@@ -18,6 +18,10 @@
 import tempfile
 import zipfile
 
+# TODO temporary work around to suppress FutureWarning message.
+import warnings
+warnings.simplefilter('ignore', category=FutureWarning)
+
 from . import expander
 from nnabla.utils import nnabla_pb2