Support multiple dataset in .proto

doc/requirements.txt

@@ -1,4 +1,4 @@
-nnabla==1.0.10
+nnabla==1.0.11
 sphinx-rtd-theme
 sphinxcontrib-actdiag
 sphinxcontrib-blockdiag

python/src/nnabla/utils/cli/compare_with_cpu.py

@@ -80,10 +80,12 @@ def compare_optimizer(config, parameters, config_cpu, parameters_cpu, result_arr
 
         for v, d in o.dataset_assign.items():
             let_data_to_variable(v.variable_instance, data[
-                                 di.variables.index(d)])
+                                 di.variables.index(d)],
+                                 data_name=d, variable_name=v.name)
         for v, d in o_cpu.dataset_assign.items():
             let_data_to_variable(v.variable_instance, data[
-                                 di.variables.index(d)])
+                                 di.variables.index(d)],
+                                 data_name=d, variable_name=v.name)
 
         # Generate data
         generated = {}
@@ -92,12 +94,14 @@ def compare_optimizer(config, parameters, config_cpu, parameters_cpu, result_arr
             dest_context = config.global_config.default_context if not o.forward_sequence or v not in o.forward_sequence[
                 0].inputs else None
             let_data_to_variable(v.variable_instance,
-                                 data=generated[v.name], ctx=dest_context)
+                                 data=generated[v.name], ctx=dest_context,
+                                 variable_name=v.name)
         for v, generator in o_cpu.generator_assign.items():
             dest_context = config.global_config.default_context if not o.forward_sequence or v not in o.forward_sequence[
                 0].inputs else None
             let_data_to_variable(v.variable_instance,
-                                 data=generated[v.name], ctx=dest_context)
+                                 data=generated[v.name], ctx=dest_context,
+                                 variable_name=v.name)
 
         last_max_diff = 1e-5
 

python/src/nnabla/utils/cli/forward.py

@@ -97,7 +97,7 @@ def _update_result(args, index, result, values, output_index, type_end_names, ou
                         # CSV type
                         with open(full_path, 'w') as f:
                             writer = csv.writer(f, lineterminator='\n')
-                            x = np.array(d, dtype=np.float32)
+                            x = np.array(d)
                             writer.writerows(x)
                     outputs[data_index].append(os.path.join('.', file_name))
         output_index += 1
@@ -120,17 +120,22 @@ class ForwardResult:
             vind = variables.index(d)
             if v.variable_instance.d.shape != data[vind].shape:
                 let_data_to_variable(v.variable_instance,
-                                     np.reshape(data[vind], v.variable_instance.d.shape))
+                                     np.reshape(
+                                         data[vind], v.variable_instance.d.shape),
+                                     data_name=d, variable_name=v.name)
             else:
                 let_data_to_variable(v.variable_instance,
-                                     data[vind].astype(v.variable_instance.d.dtype))
+                                     data[vind].astype(
+                                         v.variable_instance.d.dtype),
+                                     data_name=d, variable_name=v.name)
 
         # Generate data
         for v, generator in e.generator_assign.items():
             v.variable_instance.d = generator(v.shape)
 
         # Forward recursive
-        sum = [np.zeros(o.shape) for o in e.output_assign.keys()]
+        sum = [np.zeros(o.shape, dtype=o.variable_instance.d.dtype)
+               for o in e.output_assign.keys()]
         for i in range(e.num_evaluations):
             e.network.forward(e.forward_sequence)
             if e.need_back_propagation:
@@ -195,6 +200,7 @@ class ForwardConfig:
         batch_size=config.networks[0].batch_size,
         shuffle=False,
         normalize=normalize,
+        with_memory_cache=False,
         with_file_cache=False))
 
     # load dataset as csv
@@ -207,29 +213,31 @@ class ForwardConfig:
     rows = list(map(lambda row: list(map(lambda x: x if is_float(
         x) else compute_full_path(root_path, x), row)), rows))
 
-    with data_iterator() as di:
-        index = 0
-        while index < di.size:
-            data = di.next()
-            result, outputs = _forward(args, index, config, data, di.variables)
-            if index == 0:
-                for name, dim in zip(result.names, result.dims):
-                    if dim == 1:
-                        row0.append(name)
-                    else:
-                        for d in range(dim):
-                            row0.append(name + '__' + str(d))
-            for i, output in enumerate(outputs):
-                if index + i < len(rows):
-                    rows[index + i].extend(output)
-            index += len(outputs)
-            logger.log(
-                99, 'data {} / {}'.format(min([index, len(rows)]), len(rows)))
-
     with open(os.path.join(args.outdir, 'output_result.csv'), 'w') as f:
         writer = csv.writer(f, lineterminator='\n')
-        writer.writerow(row0)
-        writer.writerows(rows)
+        with data_iterator() as di:
+            index = 0
+            while index < di.size:
+                data = di.next()
+                result, outputs = _forward(
+                    args, index, config, data, di.variables)
+                if index == 0:
+                    for name, dim in zip(result.names, result.dims):
+                        if dim == 1:
+                            row0.append(name)
+                        else:
+                            for d in range(dim):
+                                row0.append(name + '__' + str(d))
+                    writer.writerow(row0)
+                for i, output in enumerate(outputs):
+                    if index + i < len(rows):
+                        import copy
+                        row = copy.deepcopy(rows[index + i])
+                        row.extend(output)
+                        writer.writerow(row)
+                index += len(outputs)
+                logger.log(
+                    99, 'data {} / {}'.format(min([index, len(rows)]), len(rows)))
 
     logger.log(99, 'Forward Completed.')
     progress(None)

python/src/nnabla/utils/cli/profile.py

@@ -23,35 +23,33 @@
 
 import nnabla as nn
 import nnabla.function as F
+from nnabla.ext_utils import import_extension_module
 from nnabla.logger import logger
 from nnabla.parameter import save_parameters
 from nnabla.utils.progress import configure_progress, progress
 from nnabla.utils.cli.utility import let_data_to_variable
 import nnabla.utils.load as load
 
 
-def profile(config, name, func, result_dict):
-    # for sync CPU/GPU
-    identity = F.Identity(config.global_config.default_context)
-    tmp_in = nn.Variable((1,))
-    tmp_out = nn.Variable((1,))
-    identity.setup([tmp_in], [tmp_out])
-
-    tmp_in.d = [0.]
-    identity.forward([tmp_in], [tmp_out])
+def profile(config, name, func, result_dict, synchromize):
+    # Warm-up
+    func()
+    synchromize()
 
     # Profile
-    start = time.time()
+    start_0 = time.time()
+    result = 0
     count = 0
-    while time.time() < start + 1.0 or count < 100:
+    while time.time() < start_0 + 1.0 or count < 100:
+        start = time.time()
         func()
+        synchromize()
+        stop = time.time()
+        result += stop - start
         count += 1
 
-    # sync CPU/GPU
-    identity.forward([tmp_in], [tmp_out])
-    data = tmp_out.d
+    t = result * 1000 / count
 
-    t = (time.time() - start) * 1000 / count
     logger.log(99, '%s %f(ms)' % (name, t))
     result_dict[name] = t
     return result_dict
@@ -74,7 +72,7 @@ def add_result(title, result_dict, result_array):
     return result_array
 
 
-def profile_optimizer(config, result_array):
+def profile_optimizer(config, result_array, synchronize):
     # Profile Training
     for opt in config.optimizers.values():
         o = opt.optimizer
@@ -83,14 +81,18 @@ def profile_optimizer(config, result_array):
         result_dict = OrderedDict()
 
         logger.log(99, 'Profiling ' + result_name + ' ...')
+        # Clear weight
+        for name, p in o.parameters.items():
+            if name[-2:] in ('/W', '/b'):
+                p.data.zero()
 
         # Load dataset
         def load_dataset():
             loaded_data = {}
             di = opt.data_iterator
             loaded_data[di] = di.next()
             return loaded_data
-        profile(config, 'load_dataset', load_dataset, result_dict)
+        profile(config, 'load_dataset', load_dataset, result_dict, synchronize)
 
         # Let data
         loaded_data = load_dataset()
@@ -103,24 +105,32 @@ def let_data():
                     print(opt.data_iterator.variables)
                     raise ValueError(
                         'Data "' + d + '" is not found in dataset.')
-                let_data_to_variable(v.variable_instance, data=data)
+                let_data_to_variable(v.variable_instance, data=data,
+                                     data_name=d, variable_name=v.name)
             profile(config, 'let_data (%s to %s)' %
-                    (d, v.name), let_data, result_dict)
+                    (d, v.name), let_data, result_dict, synchronize)
 
         # Generate data
         for v, generator in o.generator_assign.items():
             def generate_data():
                 let_data_to_variable(v.variable_instance,
-                                     data=generator(v.shape))
+                                     data=generator(v.shape),
+                                     variable_name=v.name)
             profile(config, 'generate_data (%s)' %
-                    v.name, generate_data, result_dict)
+                    v.name, generate_data, result_dict, synchronize)
 
+        '''
         # Setup (detail)
         for func in o.forward_sequence:
             def setup():
                 o.network.setup_function(func)
             profile(config, 'setup_function (%s : %s)' % (
-                func.name, func.function_instance.name), setup, result_dict)
+                func.name, func.function_instance.name), setup, result_dict, synchronize)
+        '''
+        # Warm-up
+        o.network.forward(o.forward_sequence)
+        o.network.prepare_backward(o.backward_sequence)
+        o.network.backward(o.backward_sequence)
 
         # Forward (detail)
         for func in o.forward_sequence:
@@ -129,12 +139,13 @@ def forward():
             in_place_str = ' : in_place' if func.function_instance.inplace_data(
                 0) > 0 else ''
             profile(config, 'forward_function (%s : %s%s)' % (
-                func.name, func.function_instance.name, in_place_str), forward, result_dict)
+                func.name, func.function_instance.name, in_place_str), forward, result_dict, synchronize)
 
         # Backward (detail)
         def prepare_backward():
             o.network.prepare_backward(o.backward_sequence)
-        profile(config, 'prepare_backward', prepare_backward, result_dict)
+        profile(config, 'prepare_backward',
+                prepare_backward, result_dict, synchronize)
         for seq in o.backward_sequence.sequence:
             o.network.prepare_backward(o.backward_sequence)
 
@@ -143,41 +154,42 @@ def backward():
             in_place_str = ' : in_place' if seq.func.function_instance.inplace_grad(
                 0) > 0 else ''
             profile(config, 'backward_function (%s : %s%s)' % (
-                seq.func.name, seq.func.function_instance.name, in_place_str), backward, result_dict)
+                seq.func.name, seq.func.function_instance.name, in_place_str), backward, result_dict, synchronize)
 
         # Forward (all)
         def forward_all():
             o.network.forward(o.forward_sequence)
-        profile(config, 'forward_all', forward_all, result_dict)
+        profile(config, 'forward_all', forward_all, result_dict, synchronize)
 
         # Backward (all)
         def backward_all():
             o.network.backward(o.backward_sequence)
-        profile(config, 'backward_all', backward_all, result_dict)
+        profile(config, 'backward_all', backward_all, result_dict, synchronize)
 
         # Backward (all)
         def backward_all_wo_zero_grad():
             o.network.backward(o.backward_sequence, parameter_zero_grad=False)
         profile(config, 'backward_all(wo param zero_grad)',
-                backward_all_wo_zero_grad, result_dict)
+                backward_all_wo_zero_grad, result_dict, synchronize)
 
         # Update (weight decay)
         if o.weight_decay > 0:
             def weight_decay():
                 o.solver.weight_decay(o.weight_decay)
             profile(config, 'weight_decay (%s)' %
-                    o.solver.name, weight_decay, result_dict)
+                    o.solver.name, weight_decay, result_dict, synchronize)
 
         # Update
         def update():
             o.solver.update()
-        profile(config, 'update (%s)' % o.solver.name, update, result_dict)
+        profile(config, 'update (%s)' %
+                o.solver.name, update, result_dict, synchronize)
 
         # Monitor loss
         def monitor_loss():
             for l in o.loss_variables:
                 np.mean(l.variable_instance.d)
-        profile(config, 'monitor_loss', monitor_loss, result_dict)
+        profile(config, 'monitor_loss', monitor_loss, result_dict, synchronize)
 
         result_array = add_result(result_name, result_dict, result_array)
 
@@ -215,14 +227,20 @@ class MonConfig:
         m.data_iterator = None
         config.monitors[name] = m
 
+    ext_module = import_extension_module(
+        config.global_config.default_context.backend[0].split(':')[0])
+
+    def synchronize(): return ext_module.synchronize(
+        device_id=config.global_config.default_context.device_id)
+
     result_array = [['time in ms']]
 
     # Profile Optimizer
     with ExitStack() as stack:
         for name, o in config.optimizers.items():
             o.data_iterator = stack.enter_context(
                 o.optimizer.data_iterator())
-        result_array = profile_optimizer(config, result_array)
+        result_array = profile_optimizer(config, result_array, synchronize)
 
     # Write profiling result
     import csv

python/src/nnabla/utils/cli/train.py

@@ -78,9 +78,7 @@ def _save_parameters(args, suffix, epoch, force=False):
     if suffix == 'best':
         base = os.path.join(args.outdir, 'results')
     filename = base + '.nnp'
-
-    if not os.path.exists(filename) and \
-       (force or timediff > 180.0 or epochdiff > 10):
+    if force or timediff > 180.0 or epochdiff > 10:
 
         version_filename = base + '_version.txt'
 
@@ -134,14 +132,16 @@ def _sum_cost():
             dest_context = config.global_config.default_context if not o.forward_sequence or v not in o.forward_sequence[
                 0].inputs else None
             let_data_to_variable(v.variable_instance, data[
-                                 di.variables.index(d)], ctx=dest_context)
+                                 di.variables.index(d)], ctx=dest_context,
+                                 data_name=d, variable_name=v.name)
 
         # Generate data
         for v, generator in o.generator_assign.items():
             dest_context = config.global_config.default_context if not o.forward_sequence or v not in o.forward_sequence[
                 0].inputs else None
             let_data_to_variable(v.variable_instance,
-                                 data=generator(v.shape), ctx=dest_context)
+                                 data=generator(v.shape), ctx=dest_context,
+                                 variable_name=v.name)
 
         # Monitor loss before forward to prepare input data while processing on
         # GPU
@@ -227,14 +227,16 @@ def _sum_error(sum, error):
                 dest_context = config.global_config.default_context if not m.forward_sequence or v not in m.forward_sequence[
                     0].inputs else None
                 let_data_to_variable(v.variable_instance, data[
-                                     di.variables.index(d)], ctx=dest_context)
+                                     di.variables.index(d)], ctx=dest_context,
+                                     data_name=d, variable_name=v.name)
 
             # Generate data
             for v, generator in m.generator_assign.items():
                 dest_context = config.global_config.default_context if not m.forward_sequence or v not in m.forward_sequence[
                     0].inputs else None
                 let_data_to_variable(v.variable_instance,
-                                     data=generator(v.shape), ctx=dest_context)
+                                     data=generator(v.shape), ctx=dest_context,
+                                     variable_name=v.name)
 
             # Sum error before forward to prepare input data while processing
             # on GPU

python/src/nnabla/utils/cli/uploader.py

@@ -73,7 +73,7 @@ def createTemporaryTar(self, name, csv_data, data_files, tmpdir):
 
         self._log('Create index.csv')
         self._progress.init(len(csv_data), 'Create index.csv')
-        with open(indexcsvfilename, 'w') as f:
+        with open(indexcsvfilename, 'w', newline='') as f:
             csvwriter = csv.writer(f)
             for row in csv_data:
                 csvwriter.writerow(row)