diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ba49c56 --- /dev/null +++ b/.gitignore @@ -0,0 +1,124 @@ +# vim swp files +*.swp +# caffe/pytorch model files +*.pth + +# Mkdocs +/docs/ +/mkdocs/docs/temp + +.DS_Store +.idea +.pytest_cache +/experiments + +# resource temp folder +tests/resources/temp/* +!tests/resources/temp/.gitkeep + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +.static_storage/ +.media/ +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +examples/text_cnn/glove_embedding/ diff --git a/README.md b/README.md index 6577bf6..88d46f7 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,3 @@ # autokeras-algorithm Some other AutoML algorithms as baselines. +Refer to: https://autokeras.com/temp/nas/ diff --git a/examples/cifar10_tutorial.py b/examples/cifar10_tutorial.py new file mode 100644 index 0000000..e398bfd --- /dev/null +++ b/examples/cifar10_tutorial.py @@ -0,0 +1,60 @@ +""" +Run NAS baseline methods +======================== +We provide 4 NAS baseline methods now, the default one is bayesian optimization. +Here is a tutorial about running NAS baseline methods. + +Generally, to run a non-default NAS methods, we will do the following steps in order: +1. Prepare the dataset in the form of torch.utils.data.DataLoader. +2. Initialize the CnnModule/MlpModule with the class name of the NAS Searcher. +3. Start search by running fit function. +Refer the cifar10 example below for more details. +""" +import numpy as np +import torch +import torchvision +import torchvision.transforms as transforms +from torch.nn.functional import cross_entropy + +from autokeras import CnnModule +from autokeras.nn.metric import Accuracy +from nas.greedy import GreedySearcher + +if __name__ == '__main__': + print('==> Preparing data..') + transform_train = transforms.Compose([ + transforms.RandomCrop(32, padding=4), + transforms.RandomHorizontalFlip(), + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), + ]) + + transform_test = transforms.Compose([ + transforms.ToTensor(), + transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)), + ]) + + trainset = torchvision.datasets.CIFAR10(root='./data', train=True, + download=True, transform=transform_train) + trainloader = torch.utils.data.DataLoader(trainset, batch_size=4, + shuffle=True, num_workers=2) + + testset = torchvision.datasets.CIFAR10(root='./data', train=False, + download=True, transform=transform_test) + testloader = torch.utils.data.DataLoader(testset, batch_size=4, + shuffle=False, num_workers=2) + (image, target) = trainset[0] + image = np.array(image).transpose((1, 2, 0)) + # add dim for batch + input_shape = np.expand_dims(image, axis=0).shape + num_classes = 10 + + # take GreedySearcher as an example, you can implement your own searcher and + # pass the class name to the CnnModule by search_type=YOUR_SEARCHER. + cnnModule = CnnModule(loss=cross_entropy, metric=Accuracy, + searcher_args={}, verbose=True, + search_type=GreedySearcher) + + cnnModule.fit(n_output_node=num_classes, input_shape=input_shape, + train_data=trainloader, + test_data=testloader) \ No newline at end of file diff --git a/nas/__init__.py b/nas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/nas/greedy.py b/nas/greedy.py new file mode 100644 index 0000000..d96a244 --- /dev/null +++ b/nas/greedy.py @@ -0,0 +1,97 @@ +import time +from copy import deepcopy + +from autokeras.custom_queue import Queue +from autokeras.bayesian import contain, SearchTree +from autokeras.net_transformer import transform +from autokeras.search import Searcher + +class GreedyOptimizer: + + def __init__(self, searcher, metric): + self.searcher = searcher + self.metric = metric + + def generate(self, descriptors, timeout, sync_message): + """Generate new neighbor architectures from the best model. + + Args: + descriptors: All the searched neural architectures. + timeout: An integer. The time limit in seconds. + sync_message: the Queue for multiprocessing return value. + + Returns: + out: A list of 2-elements tuple. Each tuple contains + an instance of Graph, a morphed neural network with weights + and the father node id in the search tree. + """ + out = [] + start_time = time.time() + descriptors = deepcopy(descriptors) + + if isinstance(sync_message, Queue) and sync_message.qsize() != 0: + return out + model_id = self.searcher.get_neighbour_best_model_id() + graph = self.searcher.load_model_by_id(model_id) + father_id = model_id + for temp_graph in transform(graph): + if contain(descriptors, temp_graph.extract_descriptor()): + continue + out.append((deepcopy(temp_graph), father_id)) + remaining_time = timeout - (time.time() - start_time) + + if remaining_time < 0: + raise TimeoutError + return out + + +class GreedySearcher(Searcher): + """ Class to search for neural architectures using Greedy search strategy. + + Attribute: + optimizer: An instance of BayesianOptimizer. + """ + + def __init__(self, n_output_node, input_shape, path, metric, loss, generators, verbose, + trainer_args=None, + default_model_len=None, + default_model_width=None): + super(GreedySearcher, self).__init__(n_output_node, input_shape, + path, metric, loss, generators, + verbose, trainer_args, default_model_len, + default_model_width) + self.optimizer = GreedyOptimizer(self, metric) + + def generate(self, multiprocessing_queue): + """Generate the next neural architecture. + + Args: + multiprocessing_queue: the Queue for multiprocessing return value. + pass into the search algorithm for synchronizing + + Returns: + results: A list of 2-element tuples. Each tuple contains an instance of Graph, + and anything to be saved in the training queue together with the architecture + + """ + remaining_time = self._timeout - time.time() + results = self.optimizer.generate(self.descriptors, remaining_time, + multiprocessing_queue) + if not results: + new_father_id = 0 + generated_graph = self.generators[0](self.n_classes, self.input_shape). \ + generate(self.default_model_len, self.default_model_width) + results.append((generated_graph, new_father_id)) + + return results + + def update(self, other_info, model_id, graph, metric_value): + return + + def load_neighbour_best_model(self): + return self.load_model_by_id(self.get_neighbour_best_model_id()) + + def get_neighbour_best_model_id(self): + if self.metric.higher_better(): + return max(self.neighbour_history, key=lambda x: x['metric_value'])['model_id'] + return min(self.neighbour_history, key=lambda x: x['metric_value'])['model_id'] diff --git a/nas/grid.py b/nas/grid.py new file mode 100644 index 0000000..dd384c5 --- /dev/null +++ b/nas/grid.py @@ -0,0 +1,97 @@ +import itertools + +from autokeras.constant import Constant +from autokeras.search import Searcher + + +def assert_search_space(search_space): + grid = search_space + value_list = [] + if Constant.LENGTH_DIM not in list(grid.keys()): + print('No length dimension found in search Space. Using default values') + grid[Constant.LENGTH_DIM] = Constant.DEFAULT_LENGTH_SEARCH + elif not isinstance(grid[Constant.LENGTH_DIM][0], int): + print('Converting String to integers. Next time please make sure to enter integer values for Length Dimension') + grid[Constant.LENGTH_DIM] = list(map(int, grid[Constant.LENGTH_DIM])) + + if Constant.WIDTH_DIM not in list(grid.keys()): + print('No width dimension found in search Space. Using default values') + grid[Constant.WIDTH_DIM] = Constant.DEFAULT_WIDTH_SEARCH + elif not isinstance(grid[Constant.WIDTH_DIM][0], int): + print('Converting String to integers. Next time please make sure to enter integer values for Width Dimension') + grid[Constant.WIDTH_DIM] = list(map(int, grid[Constant.WIDTH_DIM])) + + grid_key_list = list(grid.keys()) + grid_key_list.sort() + for key in grid_key_list: + value_list.append(grid[key]) + + dimension = list(itertools.product(*value_list)) + # print(dimension) + return grid, dimension + + +class GridSearcher(Searcher): + """ Class to search for neural architectures using Greedy search strategy. + + Attribute: + search_space: A dictionary. Specifies the search dimensions and their possible values + """ + + def __init__(self, n_output_node, input_shape, path, metric, loss, generators, verbose, search_space={}, + trainer_args=None, default_model_len=None, default_model_width=None): + super(GridSearcher, self).__init__(n_output_node, input_shape, path, metric, loss, generators, verbose, + trainer_args, default_model_len, default_model_width) + self.search_space, self.search_dimensions = assert_search_space(search_space) + self.search_space_counter = 0 + + def get_search_dimensions(self): + return self.search_dimensions + + def search_space_exhausted(self): + """ Check if Grid search has exhausted the search space """ + if self.search_space_counter == len(self.search_dimensions): + return True + return False + + def search(self, train_data, test_data, timeout=60 * 60 * 24): + """Run the search loop of training, generating and updating once. + + Call the base class implementation for search with + + Args: + train_data: An instance of DataLoader. + test_data: An instance of Dataloader. + timeout: An integer, time limit in seconds. + """ + if self.search_space_exhausted(): + return + else: + super().search(train_data, test_data, timeout) + + def update(self, other_info, model_id, graph, metric_value): + return + + def generate(self, multiprocessing_queue): + """Generate the next neural architecture. + + Args: + multiprocessing_queue: the Queue for multiprocessing return value. + + Returns: + list of 2-element tuples: generated_graph and other_info, + for grid searcher the length of list is 1. + generated_graph: An instance of Graph. + other_info: Always 0. + """ + grid = self.get_grid() + self.search_space_counter += 1 + generated_graph = self.generators[0](self.n_classes, self.input_shape). \ + generate(grid[Constant.LENGTH_DIM], grid[Constant.WIDTH_DIM]) + return [(generated_graph, 0)] + + def get_grid(self): + """ Return the next grid to be searched """ + if self.search_space_counter < len(self.search_dimensions): + return self.search_dimensions[self.search_space_counter] + return None diff --git a/nas/random.py b/nas/random.py new file mode 100644 index 0000000..1f394a6 --- /dev/null +++ b/nas/random.py @@ -0,0 +1,52 @@ +from random import randrange + +from autokeras.bayesian import SearchTree, contain +from autokeras.net_transformer import transform +from autokeras.search import Searcher + + +class RandomSearcher(Searcher): + """ Class to search for neural architectures using Random search strategy. + """ + + def __init__(self, n_output_node, input_shape, path, metric, loss, generators, verbose, + trainer_args=None, + default_model_len=None, + default_model_width=None): + super(RandomSearcher, self).__init__(n_output_node, input_shape, + path, metric, loss, generators, + verbose, trainer_args, default_model_len, + default_model_width) + + def generate(self, multiprocessing_queue): + """Generate the next neural architecture. + + Args: + multiprocessing_queue: the Queue for multiprocessing return value. + + Returns: + list of 2-element tuples: generated_graph and other_info, + for random searcher the length of list is 1. + generated_graph: An instance of Graph. + other_info: Anything to be saved in the training queue together with the architecture. + + """ + random_index = randrange(len(self.history)) + model_id = self.history[random_index]['model_id'] + graph = self.load_model_by_id(model_id) + new_father_id = None + generated_graph = None + for temp_graph in transform(graph): + if not contain(self.descriptors, temp_graph.extract_descriptor()): + new_father_id = model_id + generated_graph = temp_graph + break + if new_father_id is None: + new_father_id = 0 + generated_graph = self.generators[0](self.n_classes, self.input_shape). \ + generate(self.default_model_len, self.default_model_width) + + return [(generated_graph, new_father_id)] + + def update(self, other_info, model_id, graph, metric_value): + return \ No newline at end of file diff --git a/tests/test_nas.py b/tests/test_nas.py new file mode 100644 index 0000000..fd2df61 --- /dev/null +++ b/tests/test_nas.py @@ -0,0 +1,104 @@ +from unittest.mock import patch + +from autokeras.bayesian import edit_distance +from autokeras.backend.torch.loss_function import classification_loss +from autokeras.nn.metric import Accuracy +from autokeras.search import * +from autokeras.nn.generator import CnnGenerator, MlpGenerator, ResNetGenerator +from tests.common import clean_dir, MockProcess, get_classification_data_loaders, get_classification_data_loaders_mlp, \ + simple_transform, TEST_TEMP_DIR, simple_transform_mlp, mock_train, mock_out_of_memory_train, \ + mock_exception_handling_train + +from nas.greedy import GreedySearcher +from nas.grid import GridSearcher +from nas.random import RandomSearcher + + +@patch('torch.multiprocessing.get_context', side_effect=MockProcess) +@patch('autokeras.bayesian.transform', side_effect=simple_transform) +@patch('autokeras.backend.torch.model_trainer.ModelTrainer.train_model', side_effect=mock_train) +def test_greedy_searcher(_, _1, _2): + train_data, test_data = get_classification_data_loaders() + clean_dir(TEST_TEMP_DIR) + searcher = GreedySearcher(3, (28, 28, 3), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy, + loss=classification_loss, generators=[CnnGenerator, CnnGenerator]) + for _ in range(2): + searcher.search(train_data, test_data) + clean_dir(TEST_TEMP_DIR) + assert len(searcher.history) == 2 + + +@patch('torch.multiprocessing.get_context', side_effect=MockProcess) +@patch('autokeras.bayesian.transform', side_effect=simple_transform) +@patch('autokeras.backend.torch.model_trainer.ModelTrainer.train_model', side_effect=mock_train) +@patch('autokeras.search.get_system', return_value=Constant.SYS_GOOGLE_COLAB) +def test_greedy_searcher_sp(_, _1, _2, _3): + train_data, test_data = get_classification_data_loaders() + clean_dir(TEST_TEMP_DIR) + searcher = GreedySearcher(3, (28, 28, 3), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy, + loss=classification_loss, generators=[CnnGenerator, CnnGenerator]) + for _ in range(2): + searcher.search(train_data, test_data) + clean_dir(TEST_TEMP_DIR) + assert len(searcher.history) == 2 + + +@patch('torch.multiprocessing.get_context', side_effect=MockProcess) +@patch('autokeras.bayesian.transform', side_effect=simple_transform_mlp) +@patch('autokeras.backend.torch.model_trainer.ModelTrainer.train_model', side_effect=mock_train) +def test_greedy_searcher_mlp(_, _1, _2): + train_data, test_data = get_classification_data_loaders_mlp() + clean_dir(TEST_TEMP_DIR) + generator = GreedySearcher(3, (28,), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy, + loss=classification_loss, generators=[MlpGenerator, MlpGenerator]) + for _ in range(2): + generator.search(train_data, test_data) + clean_dir(TEST_TEMP_DIR) + assert len(generator.history) == 2 + + +@patch('torch.multiprocessing.get_context', side_effect=MockProcess) +@patch('autokeras.bayesian.transform', side_effect=simple_transform) +@patch('autokeras.backend.torch.model_trainer.ModelTrainer.train_model', side_effect=mock_train) +def test_random_searcher(_, _1, _2): + train_data, test_data = get_classification_data_loaders() + clean_dir(TEST_TEMP_DIR) + searcher = RandomSearcher(3, (28, 28, 3), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy, + loss=classification_loss, generators=[CnnGenerator, CnnGenerator]) + for _ in range(2): + searcher.search(train_data, test_data) + clean_dir(TEST_TEMP_DIR) + assert len(searcher.history) == 2 + + +@patch('torch.multiprocessing.get_context', side_effect=MockProcess) +@patch('autokeras.bayesian.transform', side_effect=simple_transform) +@patch('autokeras.backend.torch.model_trainer.ModelTrainer.train_model', side_effect=mock_train) +@patch('autokeras.search.get_system', return_value=Constant.SYS_GOOGLE_COLAB) +def test_random_searcher_sp(_, _1, _2, _3): + train_data, test_data = get_classification_data_loaders() + clean_dir(TEST_TEMP_DIR) + searcher = RandomSearcher(3, (28, 28, 3), verbose=False, path=TEST_TEMP_DIR, metric=Accuracy, + loss=classification_loss, generators=[CnnGenerator, CnnGenerator]) + for _ in range(2): + searcher.search(train_data, test_data) + clean_dir(TEST_TEMP_DIR) + assert len(searcher.history) == 2 + + +@patch('torch.multiprocessing.get_context', side_effect=MockProcess) +@patch('autokeras.bayesian.transform', side_effect=simple_transform) +@patch('autokeras.backend.torch.model_trainer.ModelTrainer.train_model', side_effect=mock_train) +def test_grid_searcher(_, _1, _2): + train_data, test_data = get_classification_data_loaders() + clean_dir(TEST_TEMP_DIR) + searcher = GridSearcher(3, (28, 28, 3), verbose=True, path=TEST_TEMP_DIR, metric=Accuracy, + loss=classification_loss, generators=[CnnGenerator, CnnGenerator]) + Constant.N_NEIGHBOURS = 1 + Constant.T_MIN = 0.8 + print(len(searcher.get_search_dimensions())) + for _ in range(len(searcher.get_search_dimensions())): + searcher.search(train_data, test_data) + clean_dir(TEST_TEMP_DIR) + assert len(searcher.history) == len(searcher.search_dimensions) +