From 1769c8f37742d152a17b38f9fa2a1849c61fd2bc Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Wed, 28 Dec 2022 11:41:22 -0800 Subject: [PATCH 1/7] Update NAS tests Signed-off-by: Maciej Szankin --- test/nas/test_nas.py | 140 +++++++++++++++++++------------------------ 1 file changed, 62 insertions(+), 78 deletions(-) diff --git a/test/nas/test_nas.py b/test/nas/test_nas.py index 10673939388..ec86fab2fd6 100644 --- a/test/nas/test_nas.py +++ b/test/nas/test_nas.py @@ -144,85 +144,69 @@ def test_basic_nas(self): best_model_archs = nas_agent() self.assertTrue(len(best_model_archs) > 0) - # Customized train, evaluation - datasets = Datasets('pytorch') - dummy_dataset = datasets['dummy'](shape=(32, 3, 64, 64), low=0., high=1., label=True) - dummy_dataloader = PyTorchDataLoader(dummy_dataset) - def train_func(model): - epochs = 2 - iters = 10 - criterion = torch.nn.CrossEntropyLoss() - optimizer = torch.optim.SGD(model.parameters(), lr=0.0001) - for nepoch in range(epochs): - model.train() - cnt = 0 - for image, target in dummy_dataloader: - print('.', end='') - cnt += 1 - output = model(image).unsqueeze(dim=0) - loss = criterion(output, target) - optimizer.zero_grad() - loss.backward() - optimizer.step() - if cnt >= iters: - break - def eval_func(model): - model.eval() - acc = 0 - for image, target in dummy_dataloader: - output = model(image).cpu().detach().numpy() - acc += np.sum(output==target) - return {'acc': acc / len(dummy_dataset)} - - for approach, search_algorithm in [(None, None), ('basic', 'grid'), ('basic', 'random'), ('basic', 'bo')]: - print('{fix}Search algorithm: {msg}{fix}'.format(msg=search_algorithm, fix='='*30)) - search_space = {'channels': [16, 32], 'dimensions': [32]} - nas_config = NASConfig(approach=approach, search_space=search_space, search_algorithm=search_algorithm) - nas_config.usr_cfg.model.framework = 'pytorch' - nas_agent = NAS(nas_config) - nas_agent.model_builder = model_builder - nas_agent.train_func = train_func - nas_agent.eval_func = eval_func - best_model_archs = nas_agent() - self.assertTrue(len(best_model_archs) > 0) - - def test_dynas(self): + def test_parameter_manager_onehot_generic(self): nas_agent = NAS('dynas_fake.yaml') - for search_algorithm, supernet in [('nsga2','ofa_mbv3_d234_e346_k357_w1.2'), ('age', 'ofa_mbv3_d234_e346_k357_w1.2')]: - config = NASConfig(approach='dynas', search_algorithm=search_algorithm) - config.dynas.supernet = supernet - config.seed = 42 - config.dynas.metrics = ['acc', 'macs', 'lat'] - config.dynas.population = 10 - config.dynas.num_evals = 10 - config.dynas.results_csv_path = 'search_results.csv' - config.dynas.batch_size = 64 - nas_agent = NAS(config) - best_model_archs = nas_agent.search() - self.assertTrue(len(best_model_archs) > 0) - - nas_agent.acc_predictor.get_parameters() - nas_agent.acc_predictor.save('tmp.pickle') - nas_agent.acc_predictor.load('tmp.pickle') - samples = nas_agent.supernet_manager.random_samples(10) - subnet_cfg = nas_agent.supernet_manager.translate2param(samples[0]) - nas_agent.runner_validate.validate_macs(subnet_cfg) - nas_agent.runner_validate.measure_latency(subnet_cfg) - nas_agent.validation_interface.clear_csv() - os.remove('tmp.pickle') - - def test_vision_reference(self): - from neural_compressor.experimental.nas.dynast.dynas_utils import \ - TorchVisionReference - reference = TorchVisionReference('ofa_mbv3', dataset_path=None, batch_size=1) - macs = reference.validate_macs() - - self.assertEqual(macs, 217234208) - - reference.measure_latency( - warmup_steps=1, - measure_steps=1, - ) + search_algorithm, supernet = 'nsga2', 'ofa_mbv3_d234_e346_k357_w1.2' + config = NASConfig(approach='dynas', search_algorithm=search_algorithm) + config.dynas.supernet = supernet + nas_agent = NAS(config) + nas_agent.init_for_search() + + pymoo_vector = [ + 1, 2, 2, 2, 2, 2, 1, 2, 0, 2, 1, 1, 0, 0, 1, + 1, 2, 2, 1, 0, 1, 1, 2, 1, 0, 1, 0, 2, 2, 2, + 0, 0, 2, 2, 2, 2, 1, 1, 2, 1, 2, 0, 2, 0, 0, + ] + onehot_vector_expected = [0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0] + + onehot_vector = nas_agent.supernet_manager.onehot_generic(in_array=pymoo_vector) + self.assertListEqual(list(onehot_vector), onehot_vector_expected) + + def test_parameter_manager_translate2param(self): + nas_agent = NAS('dynas_fake.yaml') + search_algorithm, supernet = 'nsga2', 'ofa_mbv3_d234_e346_k357_w1.2' + config = NASConfig(approach='dynas', search_algorithm=search_algorithm) + config.dynas.supernet = supernet + nas_agent = NAS(config) + nas_agent.init_for_search() + + pymoo_vector = [ + 1, 2, 2, 2, 2, 2, 1, 2, 0, 2, 1, 1, 0, 0, 1, + 1, 2, 2, 1, 0, 1, 1, 2, 1, 0, 1, 0, 2, 2, 2, + 0, 0, 2, 2, 2, 2, 1, 1, 2, 1, 2, 0, 2, 0, 0, + ] + + param_dict = nas_agent.supernet_manager.translate2param(pymoo_vector) + param_dict_expected = { + 'd': [4, 2, 4, 2, 2], + 'e': [4, 4, 6, 4, 3, 4, 3, 6, 6, 6, 3, 3, 6, 6, 6, 6, 4, 4, 6, 4], + 'ks': [5, 7, 7, 7, 7, 7, 5, 7, 3, 7, 5, 5, 3, 3, 5, 5, 7, 7, 5, 3], + } + + self.assertDictEqual(param_dict, param_dict_expected) + + + def test_parameter_manager_translate2pymoo(self): + nas_agent = NAS('dynas_fake.yaml') + search_algorithm, supernet = 'nsga2', 'ofa_mbv3_d234_e346_k357_w1.2' + config = NASConfig(approach='dynas', search_algorithm=search_algorithm) + config.dynas.supernet = supernet + nas_agent = NAS(config) + nas_agent.init_for_search() + + param_dict = { + 'd': [4, 2, 4, 2, 2], + 'e': [4, 4, 6, 4, 3, 4, 3, 6, 6, 6, 3, 3, 6, 6, 6, 6, 4, 4, 6, 4], + 'ks': [5, 7, 7, 7, 7, 7, 5, 7, 3, 7, 5, 5, 3, 3, 5, 5, 7, 7, 5, 3], + } + pymoo_vector_expected = [ + 1, 2, 2, 2, 2, 2, 1, 2, 0, 2, 1, 1, 0, 0, 1, + 1, 2, 2, 1, 0, 1, 1, 2, 1, 0, 1, 0, 2, 2, 2, + 0, 0, 2, 2, 2, 2, 1, 1, 2, 1, 2, 0, 2, 0, 0, + ] + + pymoo_vector = nas_agent.supernet_manager.translate2pymoo(param_dict) + self.assertListEqual(pymoo_vector, pymoo_vector_expected) if __name__ == "__main__": From 0a10356879401a52fabc40c6af22d1d08a840095 Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Wed, 28 Dec 2022 11:52:17 -0800 Subject: [PATCH 2/7] Update NAS tests Signed-off-by: Maciej Szankin --- test/nas/test_nas.py | 83 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/test/nas/test_nas.py b/test/nas/test_nas.py index ec86fab2fd6..6bcbfd3075d 100644 --- a/test/nas/test_nas.py +++ b/test/nas/test_nas.py @@ -1,8 +1,6 @@ import os import shutil import unittest -from pathlib import Path - import numpy as np import torch @@ -144,6 +142,87 @@ def test_basic_nas(self): best_model_archs = nas_agent() self.assertTrue(len(best_model_archs) > 0) + # Customized train, evaluation + datasets = Datasets('pytorch') + dummy_dataset = datasets['dummy'](shape=(32, 3, 64, 64), low=0., high=1., label=True) + dummy_dataloader = PyTorchDataLoader(dummy_dataset) + def train_func(model): + epochs = 2 + iters = 10 + criterion = torch.nn.CrossEntropyLoss() + optimizer = torch.optim.SGD(model.parameters(), lr=0.0001) + for nepoch in range(epochs): + model.train() + cnt = 0 + for image, target in dummy_dataloader: + print('.', end='') + cnt += 1 + output = model(image).unsqueeze(dim=0) + loss = criterion(output, target) + optimizer.zero_grad() + loss.backward() + optimizer.step() + if cnt >= iters: + break + def eval_func(model): + model.eval() + acc = 0 + for image, target in dummy_dataloader: + output = model(image).cpu().detach().numpy() + acc += np.sum(output==target) + return {'acc': acc / len(dummy_dataset)} + + for approach, search_algorithm in [(None, None), ('basic', 'grid'), ('basic', 'random'), ('basic', 'bo')]: + print('{fix}Search algorithm: {msg}{fix}'.format(msg=search_algorithm, fix='='*30)) + search_space = {'channels': [16, 32], 'dimensions': [32]} + nas_config = NASConfig(approach=approach, search_space=search_space, search_algorithm=search_algorithm) + nas_config.usr_cfg.model.framework = 'pytorch' + nas_agent = NAS(nas_config) + nas_agent.model_builder = model_builder + nas_agent.train_func = train_func + nas_agent.eval_func = eval_func + best_model_archs = nas_agent() + self.assertTrue(len(best_model_archs) > 0) + + + def test_dynas(self): + nas_agent = NAS('dynas_fake.yaml') + for search_algorithm, supernet in [('nsga2','ofa_mbv3_d234_e346_k357_w1.2'), ('age', 'ofa_mbv3_d234_e346_k357_w1.2')]: + config = NASConfig(approach='dynas', search_algorithm=search_algorithm) + config.dynas.supernet = supernet + config.seed = 42 + config.dynas.metrics = ['acc', 'macs', 'lat'] + config.dynas.population = 10 + config.dynas.num_evals = 10 + config.dynas.results_csv_path = 'search_results.csv' + config.dynas.batch_size = 64 + nas_agent = NAS(config) + best_model_archs = nas_agent.search() + self.assertTrue(len(best_model_archs) > 0) + + nas_agent.acc_predictor.get_parameters() + nas_agent.acc_predictor.save('tmp.pickle') + nas_agent.acc_predictor.load('tmp.pickle') + samples = nas_agent.supernet_manager.random_samples(10) + subnet_cfg = nas_agent.supernet_manager.translate2param(samples[0]) + nas_agent.runner_validate.validate_macs(subnet_cfg) + nas_agent.runner_validate.measure_latency(subnet_cfg) + nas_agent.validation_interface.clear_csv() + os.remove('tmp.pickle') + + def test_vision_reference(self): + from neural_compressor.experimental.nas.dynast.dynas_utils import \ + TorchVisionReference + reference = TorchVisionReference('ofa_mbv3', dataset_path=None, batch_size=1) + macs = reference.validate_macs() + + self.assertEqual(macs, 217234208) + + reference.measure_latency( + warmup_steps=1, + measure_steps=1, + ) + def test_parameter_manager_onehot_generic(self): nas_agent = NAS('dynas_fake.yaml') search_algorithm, supernet = 'nsga2', 'ofa_mbv3_d234_e346_k357_w1.2' From c7f8d2163fbf865c2218da9cc26e0492f74807a3 Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Wed, 28 Dec 2022 11:53:12 -0800 Subject: [PATCH 3/7] Update NAS tests Signed-off-by: Maciej Szankin --- test/nas/test_nas.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/nas/test_nas.py b/test/nas/test_nas.py index 6bcbfd3075d..c6466821437 100644 --- a/test/nas/test_nas.py +++ b/test/nas/test_nas.py @@ -184,7 +184,6 @@ def eval_func(model): best_model_archs = nas_agent() self.assertTrue(len(best_model_archs) > 0) - def test_dynas(self): nas_agent = NAS('dynas_fake.yaml') for search_algorithm, supernet in [('nsga2','ofa_mbv3_d234_e346_k357_w1.2'), ('age', 'ofa_mbv3_d234_e346_k357_w1.2')]: From 216f963513f00391bc3da659e280e91b094521b5 Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Wed, 28 Dec 2022 12:03:26 -0800 Subject: [PATCH 4/7] Update NAS tests - test_parameter_manager_onehot_generic Signed-off-by: Maciej Szankin --- test/nas/test_nas.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/test/nas/test_nas.py b/test/nas/test_nas.py index c6466821437..b5e857cb535 100644 --- a/test/nas/test_nas.py +++ b/test/nas/test_nas.py @@ -223,22 +223,27 @@ def test_vision_reference(self): ) def test_parameter_manager_onehot_generic(self): - nas_agent = NAS('dynas_fake.yaml') - search_algorithm, supernet = 'nsga2', 'ofa_mbv3_d234_e346_k357_w1.2' - config = NASConfig(approach='dynas', search_algorithm=search_algorithm) - config.dynas.supernet = supernet - nas_agent = NAS(config) - nas_agent.init_for_search() - - pymoo_vector = [ - 1, 2, 2, 2, 2, 2, 1, 2, 0, 2, 1, 1, 0, 0, 1, - 1, 2, 2, 1, 0, 1, 1, 2, 1, 0, 1, 0, 2, 2, 2, - 0, 0, 2, 2, 2, 2, 1, 1, 2, 1, 2, 0, 2, 0, 0, + test_configs = [ + { + 'supernet': 'ofa_mbv3_d234_e346_k357_w1.2', + 'pymoo_vector': [ + 1, 2, 2, 2, 2, 2, 1, 2, 0, 2, 1, 1, 0, 0, 1, + 1, 2, 2, 1, 0, 1, 1, 2, 1, 0, 1, 0, 2, 2, 2, + 0, 0, 2, 2, 2, 2, 1, 1, 2, 1, 2, 0, 2, 0, 0, + ], + 'onehot_vector_expected': [0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0] + }, ] - onehot_vector_expected = [0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0] + for test_config in test_configs: + nas_agent = NAS('dynas_fake.yaml') + search_algorithm, supernet = 'nsga2', test_config['supernet'] + config = NASConfig(approach='dynas', search_algorithm=search_algorithm) + config.dynas.supernet = supernet + nas_agent = NAS(config) + nas_agent.init_for_search() - onehot_vector = nas_agent.supernet_manager.onehot_generic(in_array=pymoo_vector) - self.assertListEqual(list(onehot_vector), onehot_vector_expected) + onehot_vector = nas_agent.supernet_manager.onehot_generic(in_array=test_config['pymoo_vector']) + self.assertListEqual(list(onehot_vector), test_config['onehot_vector_expected']) def test_parameter_manager_translate2param(self): nas_agent = NAS('dynas_fake.yaml') From c9ff008ef6895a685f3ebaaf0564c1d49bbae1d1 Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Wed, 28 Dec 2022 12:11:24 -0800 Subject: [PATCH 5/7] Update NAS tests - test_parameter_manager_translate2pymoo Signed-off-by: Maciej Szankin --- test/nas/test_nas.py | 46 +++++++++++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/test/nas/test_nas.py b/test/nas/test_nas.py index b5e857cb535..c13a333c5fb 100644 --- a/test/nas/test_nas.py +++ b/test/nas/test_nas.py @@ -270,26 +270,36 @@ def test_parameter_manager_translate2param(self): def test_parameter_manager_translate2pymoo(self): - nas_agent = NAS('dynas_fake.yaml') - search_algorithm, supernet = 'nsga2', 'ofa_mbv3_d234_e346_k357_w1.2' - config = NASConfig(approach='dynas', search_algorithm=search_algorithm) - config.dynas.supernet = supernet - nas_agent = NAS(config) - nas_agent.init_for_search() - - param_dict = { - 'd': [4, 2, 4, 2, 2], - 'e': [4, 4, 6, 4, 3, 4, 3, 6, 6, 6, 3, 3, 6, 6, 6, 6, 4, 4, 6, 4], - 'ks': [5, 7, 7, 7, 7, 7, 5, 7, 3, 7, 5, 5, 3, 3, 5, 5, 7, 7, 5, 3], - } - pymoo_vector_expected = [ - 1, 2, 2, 2, 2, 2, 1, 2, 0, 2, 1, 1, 0, 0, 1, - 1, 2, 2, 1, 0, 1, 1, 2, 1, 0, 1, 0, 2, 2, 2, - 0, 0, 2, 2, 2, 2, 1, 1, 2, 1, 2, 0, 2, 0, 0, + test_configs = [ + { + 'supernet': 'ofa_mbv3_d234_e346_k357_w1.2', + 'param_dict': { + 'd': [4, 2, 4, 2, 2], + 'e': [4, 4, 6, 4, 3, 4, 3, 6, 6, 6, 3, 3, 6, 6, 6, 6, 4, 4, 6, 4], + 'ks': [5, 7, 7, 7, 7, 7, 5, 7, 3, 7, 5, 5, 3, 3, 5, 5, 7, 7, 5, 3], + }, + 'pymoo_vector_expected': [ + 1, 2, 2, 2, 2, 2, 1, 2, 0, 2, 1, 1, 0, 0, 1, + 1, 2, 2, 1, 0, 1, 1, 2, 1, 0, 1, 0, 2, 2, 2, + 0, 0, 2, 2, 2, 2, 1, 1, 2, 1, 2, 0, 2, 0, 0, + ] + }, + { + 'supernet': 'transformer_lt_wmt_en_de', + 'param_dict': {'encoder_embed_dim': [640], 'decoder_embed_dim': [640], 'encoder_ffn_embed_dim': [1024, 2048, 3072, 3072, 3072, 1024], 'decoder_ffn_embed_dim': [3072, 1024, 1024, 1024, 3072, 1024], 'decoder_layer_num': [3], 'encoder_self_attention_heads': [8, 8, 8, 8, 8, 8], 'decoder_self_attention_heads': [8, 4, 8, 4, 4, 8], 'decoder_ende_attention_heads': [8, 4, 4, 4, 8, 8], 'decoder_arbitrary_ende_attn': [1, -1, -1, 1, -1, 1]}, + 'pymoo_vector_expected': [0, 0, 2, 1, 0, 0, 0, 2, 0, 2, 2, 2, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1], + } ] + for test_config in test_configs: + nas_agent = NAS('dynas_fake.yaml') + search_algorithm, supernet = 'nsga2', test_config['supernet'] + config = NASConfig(approach='dynas', search_algorithm=search_algorithm) + config.dynas.supernet = supernet + nas_agent = NAS(config) + nas_agent.init_for_search() - pymoo_vector = nas_agent.supernet_manager.translate2pymoo(param_dict) - self.assertListEqual(pymoo_vector, pymoo_vector_expected) + pymoo_vector = nas_agent.supernet_manager.translate2pymoo(test_config['param_dict']) + self.assertListEqual(pymoo_vector, test_config['pymoo_vector_expected']) if __name__ == "__main__": From 90b96c726fbaf3bae96d65c521889ca5a949a76b Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Wed, 28 Dec 2022 12:32:17 -0800 Subject: [PATCH 6/7] Update NAS tests Signed-off-by: Maciej Szankin --- test/nas/test_nas.py | 54 ++++++++++++++++++++++++++++---------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/test/nas/test_nas.py b/test/nas/test_nas.py index c13a333c5fb..1cb7bebf3ca 100644 --- a/test/nas/test_nas.py +++ b/test/nas/test_nas.py @@ -233,6 +233,11 @@ def test_parameter_manager_onehot_generic(self): ], 'onehot_vector_expected': [0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0] }, + { + 'supernet': 'transformer_lt_wmt_en_de', + 'pymoo_vector': [0, 0, 2, 1, 0, 0, 0, 2, 0, 2, 2, 2, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1], + 'onehot_vector_expected': [1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0] + }, ] for test_config in test_configs: nas_agent = NAS('dynas_fake.yaml') @@ -246,27 +251,38 @@ def test_parameter_manager_onehot_generic(self): self.assertListEqual(list(onehot_vector), test_config['onehot_vector_expected']) def test_parameter_manager_translate2param(self): - nas_agent = NAS('dynas_fake.yaml') - search_algorithm, supernet = 'nsga2', 'ofa_mbv3_d234_e346_k357_w1.2' - config = NASConfig(approach='dynas', search_algorithm=search_algorithm) - config.dynas.supernet = supernet - nas_agent = NAS(config) - nas_agent.init_for_search() - - pymoo_vector = [ - 1, 2, 2, 2, 2, 2, 1, 2, 0, 2, 1, 1, 0, 0, 1, - 1, 2, 2, 1, 0, 1, 1, 2, 1, 0, 1, 0, 2, 2, 2, - 0, 0, 2, 2, 2, 2, 1, 1, 2, 1, 2, 0, 2, 0, 0, + test_configs = [ + { + 'supernet': 'ofa_mbv3_d234_e346_k357_w1.2', + 'pymoo_vector': [ + 1, 2, 2, 2, 2, 2, 1, 2, 0, 2, 1, 1, 0, 0, 1, + 1, 2, 2, 1, 0, 1, 1, 2, 1, 0, 1, 0, 2, 2, 2, + 0, 0, 2, 2, 2, 2, 1, 1, 2, 1, 2, 0, 2, 0, 0, + ], + 'param_dict_expected': { + 'd': [4, 2, 4, 2, 2], + 'e': [4, 4, 6, 4, 3, 4, 3, 6, 6, 6, 3, 3, 6, 6, 6, 6, 4, 4, 6, 4], + 'ks': [5, 7, 7, 7, 7, 7, 5, 7, 3, 7, 5, 5, 3, 3, 5, 5, 7, 7, 5, 3], + }, + }, + { + 'supernet': 'transformer_lt_wmt_en_de', + 'pymoo_vector': [0, 0, 2, 1, 0, 0, 0, 2, 0, 2, 2, 2, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1], + 'param_dict_expected': {'encoder_embed_dim': [640], 'decoder_embed_dim': [640], 'encoder_ffn_embed_dim': [1024, 2048, 3072, 3072, 3072, 1024], 'decoder_ffn_embed_dim': [3072, 1024, 1024, 1024, 3072, 1024], 'decoder_layer_num': [3], 'encoder_self_attention_heads': [8, 8, 8, 8, 8, 8], 'decoder_self_attention_heads': [8, 4, 8, 4, 4, 8], 'decoder_ende_attention_heads': [8, 4, 4, 4, 8, 8], 'decoder_arbitrary_ende_attn': [1, -1, -1, 1, -1, 1]}, + } ] - param_dict = nas_agent.supernet_manager.translate2param(pymoo_vector) - param_dict_expected = { - 'd': [4, 2, 4, 2, 2], - 'e': [4, 4, 6, 4, 3, 4, 3, 6, 6, 6, 3, 3, 6, 6, 6, 6, 4, 4, 6, 4], - 'ks': [5, 7, 7, 7, 7, 7, 5, 7, 3, 7, 5, 5, 3, 3, 5, 5, 7, 7, 5, 3], - } + for test_config in test_configs: + nas_agent = NAS('dynas_fake.yaml') + search_algorithm, supernet = 'nsga2', test_config['supernet'] + config = NASConfig(approach='dynas', search_algorithm=search_algorithm) + config.dynas.supernet = supernet + nas_agent = NAS(config) + nas_agent.init_for_search() + + param_dict = nas_agent.supernet_manager.translate2param(test_config['pymoo_vector']) - self.assertDictEqual(param_dict, param_dict_expected) + self.assertDictEqual(param_dict, test_config['param_dict_expected']) def test_parameter_manager_translate2pymoo(self): @@ -282,7 +298,7 @@ def test_parameter_manager_translate2pymoo(self): 1, 2, 2, 2, 2, 2, 1, 2, 0, 2, 1, 1, 0, 0, 1, 1, 2, 2, 1, 0, 1, 1, 2, 1, 0, 1, 0, 2, 2, 2, 0, 0, 2, 2, 2, 2, 1, 1, 2, 1, 2, 0, 2, 0, 0, - ] + ], }, { 'supernet': 'transformer_lt_wmt_en_de', From 654b18aac5a5ab13e9bbb62f27fd8dc91c634257 Mon Sep 17 00:00:00 2001 From: Maciej Szankin Date: Wed, 28 Dec 2022 16:13:30 -0800 Subject: [PATCH 7/7] Update NAS tests - fix formatting Signed-off-by: Maciej Szankin --- test/nas/test_nas.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/test/nas/test_nas.py b/test/nas/test_nas.py index 1cb7bebf3ca..d4a604c1784 100644 --- a/test/nas/test_nas.py +++ b/test/nas/test_nas.py @@ -142,7 +142,7 @@ def test_basic_nas(self): best_model_archs = nas_agent() self.assertTrue(len(best_model_archs) > 0) - # Customized train, evaluation + # Customized train, evaluation datasets = Datasets('pytorch') dummy_dataset = datasets['dummy'](shape=(32, 3, 64, 64), low=0., high=1., label=True) dummy_dataloader = PyTorchDataLoader(dummy_dataset) @@ -231,12 +231,29 @@ def test_parameter_manager_onehot_generic(self): 1, 2, 2, 1, 0, 1, 1, 2, 1, 0, 1, 0, 2, 2, 2, 0, 0, 2, 2, 2, 2, 1, 1, 2, 1, 2, 0, 2, 0, 0, ], - 'onehot_vector_expected': [0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0] + 'onehot_vector_expected': [ + 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, + 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, + 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, + 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, + 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, + 1, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 1, + 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, + 0, 0, 1, 1, 0, 0, 1, 0, 0, + ], }, { 'supernet': 'transformer_lt_wmt_en_de', - 'pymoo_vector': [0, 0, 2, 1, 0, 0, 0, 2, 0, 2, 2, 2, 0, 2, 3, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1], - 'onehot_vector_expected': [1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0] + 'pymoo_vector': [ + 0, 0, 2, 1, 0, 0, 0, 2, 0, 2, 2, 2, 0, 2, 3, 0, 0, 0, 0, 0, + 0, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1], + 'onehot_vector_expected': [ + 1, 0, 1, 0, 0, 0, 1, 0, 1, 0, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, + 0, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, + 0, 0, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, + 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 0, + 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, + ], }, ] for test_config in test_configs: