From fb22067d3bffb0f7a512cdd1a1de28db87cf6e23 Mon Sep 17 00:00:00 2001 From: Ricocotam Date: Sat, 28 Mar 2020 19:44:58 +0100 Subject: [PATCH 1/6] minimal new helper --- bootstrap/new.py | 64 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) create mode 100644 bootstrap/new.py diff --git a/bootstrap/new.py b/bootstrap/new.py new file mode 100644 index 0000000..1db51c1 --- /dev/null +++ b/bootstrap/new.py @@ -0,0 +1,64 @@ +from pathlib import Path +from argparse import ArgumentParser + + +parser = ArgumentParser() +parser.add_argument("project_path", action="store", type=str, help="Path to new project") +parser.add_argument("project_name", type=str, help="Project name") + + +if __name__ == "__main__": + args = parser.parse_args() + path = Path(args.project_path) + + path.mkdir() + path = path / args.project_name + path.mkdir() + + Path(path / "models").mkdir() + Path(path / "models/__init__.py").touch() + Path(path / "models/networks").mkdir() + Path(path / "models/networks/__init__.py").touch() + Path(path / "models/networks/factory.py").touch() + Path(path / "models/networks/mynetwork.py").touch() + + with open(path / "models/networks/factory.py", "w") as f: + f.write(r"""from bootstrap.lib.options import Options +from bootstrap.lib.logger import Logger +from bootstrap.models.networks.data_parallel import DataParallel + +from .mynetwork import MyNetwork + + +def factory(engine): + logger = Logger() + net_opt = Options()["model"]["network"] + logger("Creating Network") + if net_opt["name"] == "mynetwork": + # You can use any param to create your network + # You just have to write them in your option file from options/ folder + net = MyNetwork(net_opt["param1"], net_opt["param2"]) + else: + raise ValueError(opt["name"]) + logger("Network was created") + + if torch.cuda.device_count() > 1: + net = DataParallel(net) + + return net +""") + + with open(path / "models/networks/mynetwork.py", "w") as f: + f.write(r"""import torch.nn as nn + + +class MyNetwork(nn.Module): + def __init__(self, *args, **kwargs): + super(MyNetwork, self).__init__() + # Assign args + + def forward(self, x): + # x is a dictionnary given by Dataset class + pred = self.net(x) + return pred # This is a tensor (or several tensors) +""") From 939ce7584568c792a151d732f253ba5d7bf8086b Mon Sep 17 00:00:00 2001 From: Ricocotam Date: Sun, 29 Mar 2020 17:57:37 +0200 Subject: [PATCH 2/6] basic template, more complete example --- bootstrap/new.py | 122 +++++++++++------- bootstrap/template/__init__.py | 0 bootstrap/template/datasets/__init__.py | 0 bootstrap/template/datasets/factory.py | 42 ++++++ .../template/datasets/template_dataset.py | 23 ++++ bootstrap/template/engines/__init__.py | 0 bootstrap/template/engines/factory.py | 0 bootstrap/template/engines/myengine.py | 0 bootstrap/template/models/__init__.py | 0 .../template/models/criterions/__init__.py | 0 .../template/models/criterions/factory.py | 16 +++ .../models/criterions/template_criterion.py | 12 ++ bootstrap/template/models/metrics/__init__.py | 0 bootstrap/template/models/metrics/factory.py | 14 ++ .../models/metrics/template_metric.py | 13 ++ .../template/models/networks/__init__.py | 0 bootstrap/template/models/networks/factory.py | 22 ++++ .../models/networks/template_network.py | 12 ++ bootstrap/template/optimizers/__init__.py | 0 bootstrap/template/optimizers/factory.py | 0 bootstrap/template/optimizers/myoptimizer.py | 0 21 files changed, 227 insertions(+), 49 deletions(-) create mode 100644 bootstrap/template/__init__.py create mode 100644 bootstrap/template/datasets/__init__.py create mode 100644 bootstrap/template/datasets/factory.py create mode 100644 bootstrap/template/datasets/template_dataset.py create mode 100644 bootstrap/template/engines/__init__.py create mode 100644 bootstrap/template/engines/factory.py create mode 100644 bootstrap/template/engines/myengine.py create mode 100644 bootstrap/template/models/__init__.py create mode 100644 bootstrap/template/models/criterions/__init__.py create mode 100644 bootstrap/template/models/criterions/factory.py create mode 100644 bootstrap/template/models/criterions/template_criterion.py create mode 100644 bootstrap/template/models/metrics/__init__.py create mode 100644 bootstrap/template/models/metrics/factory.py create mode 100644 bootstrap/template/models/metrics/template_metric.py create mode 100644 bootstrap/template/models/networks/__init__.py create mode 100644 bootstrap/template/models/networks/factory.py create mode 100644 bootstrap/template/models/networks/template_network.py create mode 100644 bootstrap/template/optimizers/__init__.py create mode 100644 bootstrap/template/optimizers/factory.py create mode 100644 bootstrap/template/optimizers/myoptimizer.py diff --git a/bootstrap/new.py b/bootstrap/new.py index 1db51c1..933aace 100644 --- a/bootstrap/new.py +++ b/bootstrap/new.py @@ -1,64 +1,88 @@ +import os from pathlib import Path from argparse import ArgumentParser +file_dir = Path(__file__).parent + + parser = ArgumentParser() -parser.add_argument("project_path", action="store", type=str, help="Path to new project") -parser.add_argument("project_name", type=str, help="Project name") +parser.add_argument("--project_name", type=str, help="Project name") + + +def get_template_file(filename, project_name): + parts = list(filename.parts) + project_index = parts.index(project_name.lower()) + if parts[-1] not in ["__init__.py", "factory.py"]: + parts[-1] = "template_" + parts[-1][2:] # remove "my" + template_path = "/".join(parts[project_index + 1:]) + template_path = file_dir / Path("template") / template_path + + return template_path + + +def get_file_content(filename, project_name): + template = get_template_file(filename, project_name) + + content = Path(template).read_text() + content = content.replace("{PROJECT_NAME}", project_name) + content = content.replace("{PROJECT_NAME_LOWER}", project_name.lower()) + content = content.replace("{PROJECT_NAME_UPPER}", project_name.upper()) + + return content + + +def write_files(files, project_name): + for f in files: + content = get_file_content(f, project_name) + f.write_text(content) + + +def get_files(directory): + dir_name = directory.stem + if dir_name == "options": + return [directory / "abstract.yaml"] + + to_ret = [] + if dir_name != "models": + to_ret.append(directory / "__init__.py") + + to_ret.append(directory / "factory.py") + custom_file = f"my{dir_name[:-1]}.py" + to_ret.append(directory / custom_file) + + return to_ret if __name__ == "__main__": args = parser.parse_args() - path = Path(args.project_path) + project_name = args.project_name + path = Path(f"{project_name.lower()}.bootstrap.pytorch") path.mkdir() - path = path / args.project_name + os.mkdir(path / "logs") + path = Path(f"{project_name.lower()}.bootstrap.pytorch/{project_name.lower()}") path.mkdir() + print(f"Creating logs directory") + + print("Creating models directory and __init__ file") Path(path / "models").mkdir() Path(path / "models/__init__.py").touch() - Path(path / "models/networks").mkdir() - Path(path / "models/networks/__init__.py").touch() - Path(path / "models/networks/factory.py").touch() - Path(path / "models/networks/mynetwork.py").touch() - - with open(path / "models/networks/factory.py", "w") as f: - f.write(r"""from bootstrap.lib.options import Options -from bootstrap.lib.logger import Logger -from bootstrap.models.networks.data_parallel import DataParallel - -from .mynetwork import MyNetwork - - -def factory(engine): - logger = Logger() - net_opt = Options()["model"]["network"] - logger("Creating Network") - if net_opt["name"] == "mynetwork": - # You can use any param to create your network - # You just have to write them in your option file from options/ folder - net = MyNetwork(net_opt["param1"], net_opt["param2"]) - else: - raise ValueError(opt["name"]) - logger("Network was created") - - if torch.cuda.device_count() > 1: - net = DataParallel(net) - - return net -""") - - with open(path / "models/networks/mynetwork.py", "w") as f: - f.write(r"""import torch.nn as nn - - -class MyNetwork(nn.Module): - def __init__(self, *args, **kwargs): - super(MyNetwork, self).__init__() - # Assign args - - def forward(self, x): - # x is a dictionnary given by Dataset class - pred = self.net(x) - return pred # This is a tensor (or several tensors) -""") + + directories = [ + "datasets", + "models/networks", + "models/criterions", + "models/metrics", + ] + + for directory in directories: + print(f"Creating {directory} folder and associated files") + new_dir = path / directory + if directory != "models": + new_dir.mkdir() + files = get_files(new_dir) + write_files(files, project_name) + + print("Project is ready !") diff --git a/bootstrap/template/__init__.py b/bootstrap/template/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap/template/datasets/__init__.py b/bootstrap/template/datasets/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap/template/datasets/factory.py b/bootstrap/template/datasets/factory.py new file mode 100644 index 0000000..8f86d76 --- /dev/null +++ b/bootstrap/template/datasets/factory.py @@ -0,0 +1,42 @@ +from bootstrap.lib.options import Options +from bootstrap.lib.logger import Logger + +from .mydataset import {PROJECT_NAME}Dataset + + +def factory(engine=None): + logger = Logger() + logger('Creating dataset...') + + opt = Options()["dataset"] + + dataset = {} + + if opt.get("train_split", None): + logger("Loading train data") + dataset["train"] = factory_split(opt["train_split"]) + logger(f"Train dataset length is {len(dataset['train'])}") + + if opt.get("eval_split", None): + logger("Loading test data") + dataset["eval"] = factory_split(opt["eval_split"]) + logger(f"Test dataset length is {len(dataset['eval'])}") + + logger("Dataset was created") + return dataset + + +def factory_split(split): + opt = Options()["dataset"] + + shuffle = ("train" in split) + + dataset = {PROJECT_NAME}Dataset( + dir_data=opt["dir"], + split=split, + batch_size=opt["batch_size"], + shuffle=shuffle, + nb_threads=opt["nb_threads"] + ) + + return dataset diff --git a/bootstrap/template/datasets/template_dataset.py b/bootstrap/template/datasets/template_dataset.py new file mode 100644 index 0000000..3a3d641 --- /dev/null +++ b/bootstrap/template/datasets/template_dataset.py @@ -0,0 +1,23 @@ +from bootstrap.datasets.dataset import Dataset + + +class {PROJECT_NAME}Dataset(Dataset): + """ Dataset of Wikipedia Comparable Article + + Parameters + ----------- + """ + def __init__(self, + dir_data, + split='train', + batch_size=4, + shuffle=False, + pin_memory=False, + nb_threads=4): + super({PROJECT_NAME}Dataset, self).__init__(dir_data, split, batch_size, shuffle, pin_memory, nb_threads) + + def __len__(self): + raise NotImplementedError + + def __getitem__(self, i): + raise NotImplementedError diff --git a/bootstrap/template/engines/__init__.py b/bootstrap/template/engines/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap/template/engines/factory.py b/bootstrap/template/engines/factory.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap/template/engines/myengine.py b/bootstrap/template/engines/myengine.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap/template/models/__init__.py b/bootstrap/template/models/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap/template/models/criterions/__init__.py b/bootstrap/template/models/criterions/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap/template/models/criterions/factory.py b/bootstrap/template/models/criterions/factory.py new file mode 100644 index 0000000..f4560b9 --- /dev/null +++ b/bootstrap/template/models/criterions/factory.py @@ -0,0 +1,16 @@ +from bootstrap.lib.options import Options +from bootstrap.lib.logger import Logger + +from .mycriterion import {PROJECT_NAME}Criterion + + +def factory(engine=None, mode=None): + logger = Logger() + logger('Creating criterion for {} mode...'.format(mode)) + + if Options()['model']['criterion'].get('import', False): + criterion = {PROJECT_NAME}Criterion() + else: + raise ValueError() + + return criterion diff --git a/bootstrap/template/models/criterions/template_criterion.py b/bootstrap/template/models/criterions/template_criterion.py new file mode 100644 index 0000000..e331f35 --- /dev/null +++ b/bootstrap/template/models/criterions/template_criterion.py @@ -0,0 +1,12 @@ +import torch.nn as nn + + +class {PROJECT_NAME}Criterion(nn.Module): + + def __init__(self): + super({PROJECT_NAME}Criterion, self).__init__() + + def forward(self, net_out, batch): + # net_out : output of network + # batch : output of dataset (after collate function) + raise NotImplementedError diff --git a/bootstrap/template/models/metrics/__init__.py b/bootstrap/template/models/metrics/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap/template/models/metrics/factory.py b/bootstrap/template/models/metrics/factory.py new file mode 100644 index 0000000..f78cb52 --- /dev/null +++ b/bootstrap/template/models/metrics/factory.py @@ -0,0 +1,14 @@ +from bootstrap.lib.options import Options + +from .mymetric import {PROJECT_NAME}Metric + + +def factory(engine=None, mode="train"): + opt = Options()['model.metric'] + + if opt['name'] == '{PROJECT_NAME_LOWER}metric': + metric = {PROJECT_NAME_LOWER}metric() + else: + raise ValueError(opt['name']) + + return metric diff --git a/bootstrap/template/models/metrics/template_metric.py b/bootstrap/template/models/metrics/template_metric.py new file mode 100644 index 0000000..b6fd58a --- /dev/null +++ b/bootstrap/template/models/metrics/template_metric.py @@ -0,0 +1,13 @@ +import torch.nn as nn +from bootstrap.lib.logger import Logger + + +class {PROJECT_NAME}Metric(nn.Module): + def __init__(self): + super(Accuracy, self).__init__() + + def forward(self, crit_out, net_out, batch): + # crit_out : output of criterion (dictionnary) + # net_out : output of network + # batch : output of dataset (after collate function) + raise NotImplementedError diff --git a/bootstrap/template/models/networks/__init__.py b/bootstrap/template/models/networks/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap/template/models/networks/factory.py b/bootstrap/template/models/networks/factory.py new file mode 100644 index 0000000..fa7e463 --- /dev/null +++ b/bootstrap/template/models/networks/factory.py @@ -0,0 +1,22 @@ +from bootstrap.lib.options import Options +from bootstrap.lib.logger import Logger +from bootstrap.models.networks.data_parallel import DataParallel + +from .mynetwork import {PROJECT_NAME}Network + + +def factory(engine): + logger = Logger() + net_opt = Options()["model"]["network"] + logger("Creating Network...") + + if net_opt["name"] == "{PROJECT_NAME_LOWER}network": + # You can use any param to create your network + # You just have to write them in your option file from options/ folder + net = {PROJECT_NAME}Network(net_opt["param1"], net_opt["param2"]) + else: + raise ValueError(opt["name"]) + logger("Network was created") + if torch.cuda.device_count() > 1: + net = DataParallel(net) + return net diff --git a/bootstrap/template/models/networks/template_network.py b/bootstrap/template/models/networks/template_network.py new file mode 100644 index 0000000..f10c7b1 --- /dev/null +++ b/bootstrap/template/models/networks/template_network.py @@ -0,0 +1,12 @@ +import torch.nn as nn + + +class {PROJECT_NAME}Network(nn.Module): + def __init__(self, *args, **kwargs): + super(MyNetwork, self).__init__() + # Assign args + + def forward(self, x): + # x is a dictionnary given by Dataset class + pred = self.net(x) + return pred # This is a tensor (or several tensors) \ No newline at end of file diff --git a/bootstrap/template/optimizers/__init__.py b/bootstrap/template/optimizers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap/template/optimizers/factory.py b/bootstrap/template/optimizers/factory.py new file mode 100644 index 0000000..e69de29 diff --git a/bootstrap/template/optimizers/myoptimizer.py b/bootstrap/template/optimizers/myoptimizer.py new file mode 100644 index 0000000..e69de29 From 5cec8b49a55b07a51688e98fd565cdacf265fe94 Mon Sep 17 00:00:00 2001 From: Adrien Pouyet Date: Wed, 1 Apr 2020 23:40:52 +0200 Subject: [PATCH 3/6] right template path and manifest include of templates --- MANIFEST.in | 3 ++- bootstrap/new.py | 7 +++++-- bootstrap/template/optimizers/__init__.py | 0 .../default/project/datasets}/__init__.py | 0 .../default/project}/datasets/factory.py | 0 .../default/project}/datasets/template_dataset.py | 0 .../default/project/engines}/__init__.py | 0 .../default/project}/engines/factory.py | 0 .../default/project}/engines/myengine.py | 0 .../default/project/models}/__init__.py | 0 .../default/project/models/criterions}/__init__.py | 0 .../default/project}/models/criterions/factory.py | 0 .../project}/models/criterions/template_criterion.py | 0 .../default/project/models/metrics}/__init__.py | 0 .../default/project}/models/metrics/factory.py | 0 .../default/project}/models/metrics/template_metric.py | 0 .../default/project/models/networks}/__init__.py | 0 .../default/project}/models/networks/factory.py | 0 .../default/project}/models/networks/template_network.py | 0 .../default/project/optimizers}/__init__.py | 0 .../default/project}/optimizers/factory.py | 0 .../default/project}/optimizers/myoptimizer.py | 0 setup.py | 1 + 23 files changed, 8 insertions(+), 3 deletions(-) delete mode 100644 bootstrap/template/optimizers/__init__.py rename bootstrap/{template => templates/default/project/datasets}/__init__.py (100%) rename bootstrap/{template => templates/default/project}/datasets/factory.py (100%) rename bootstrap/{template => templates/default/project}/datasets/template_dataset.py (100%) rename bootstrap/{template/datasets => templates/default/project/engines}/__init__.py (100%) rename bootstrap/{template => templates/default/project}/engines/factory.py (100%) rename bootstrap/{template => templates/default/project}/engines/myengine.py (100%) rename bootstrap/{template/engines => templates/default/project/models}/__init__.py (100%) rename bootstrap/{template/models => templates/default/project/models/criterions}/__init__.py (100%) rename bootstrap/{template => templates/default/project}/models/criterions/factory.py (100%) rename bootstrap/{template => templates/default/project}/models/criterions/template_criterion.py (100%) rename bootstrap/{template/models/criterions => templates/default/project/models/metrics}/__init__.py (100%) rename bootstrap/{template => templates/default/project}/models/metrics/factory.py (100%) rename bootstrap/{template => templates/default/project}/models/metrics/template_metric.py (100%) rename bootstrap/{template/models/metrics => templates/default/project/models/networks}/__init__.py (100%) rename bootstrap/{template => templates/default/project}/models/networks/factory.py (100%) rename bootstrap/{template => templates/default/project}/models/networks/template_network.py (100%) rename bootstrap/{template/models/networks => templates/default/project/optimizers}/__init__.py (100%) rename bootstrap/{template => templates/default/project}/optimizers/factory.py (100%) rename bootstrap/{template => templates/default/project}/optimizers/myoptimizer.py (100%) diff --git a/MANIFEST.in b/MANIFEST.in index cd797d8..56e7a5b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,3 @@ include LICENSE.txt -include README.md \ No newline at end of file +include README.md +recursive-include bootstrap/templates *py \ No newline at end of file diff --git a/bootstrap/new.py b/bootstrap/new.py index 933aace..ad57e4f 100644 --- a/bootstrap/new.py +++ b/bootstrap/new.py @@ -60,11 +60,14 @@ def get_files(directory): path = Path(f"{project_name.lower()}.bootstrap.pytorch") path.mkdir() + + print(f"Creating logs directory") os.mkdir(path / "logs") + + print(f"Creating project directory and __init__.py file") path = Path(f"{project_name.lower()}.bootstrap.pytorch/{project_name.lower()}") path.mkdir() - - print(f"Creating logs directory") + Path(path / "__init__.py").touch() print("Creating models directory and __init__ file") Path(path / "models").mkdir() diff --git a/bootstrap/template/optimizers/__init__.py b/bootstrap/template/optimizers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bootstrap/template/__init__.py b/bootstrap/templates/default/project/datasets/__init__.py similarity index 100% rename from bootstrap/template/__init__.py rename to bootstrap/templates/default/project/datasets/__init__.py diff --git a/bootstrap/template/datasets/factory.py b/bootstrap/templates/default/project/datasets/factory.py similarity index 100% rename from bootstrap/template/datasets/factory.py rename to bootstrap/templates/default/project/datasets/factory.py diff --git a/bootstrap/template/datasets/template_dataset.py b/bootstrap/templates/default/project/datasets/template_dataset.py similarity index 100% rename from bootstrap/template/datasets/template_dataset.py rename to bootstrap/templates/default/project/datasets/template_dataset.py diff --git a/bootstrap/template/datasets/__init__.py b/bootstrap/templates/default/project/engines/__init__.py similarity index 100% rename from bootstrap/template/datasets/__init__.py rename to bootstrap/templates/default/project/engines/__init__.py diff --git a/bootstrap/template/engines/factory.py b/bootstrap/templates/default/project/engines/factory.py similarity index 100% rename from bootstrap/template/engines/factory.py rename to bootstrap/templates/default/project/engines/factory.py diff --git a/bootstrap/template/engines/myengine.py b/bootstrap/templates/default/project/engines/myengine.py similarity index 100% rename from bootstrap/template/engines/myengine.py rename to bootstrap/templates/default/project/engines/myengine.py diff --git a/bootstrap/template/engines/__init__.py b/bootstrap/templates/default/project/models/__init__.py similarity index 100% rename from bootstrap/template/engines/__init__.py rename to bootstrap/templates/default/project/models/__init__.py diff --git a/bootstrap/template/models/__init__.py b/bootstrap/templates/default/project/models/criterions/__init__.py similarity index 100% rename from bootstrap/template/models/__init__.py rename to bootstrap/templates/default/project/models/criterions/__init__.py diff --git a/bootstrap/template/models/criterions/factory.py b/bootstrap/templates/default/project/models/criterions/factory.py similarity index 100% rename from bootstrap/template/models/criterions/factory.py rename to bootstrap/templates/default/project/models/criterions/factory.py diff --git a/bootstrap/template/models/criterions/template_criterion.py b/bootstrap/templates/default/project/models/criterions/template_criterion.py similarity index 100% rename from bootstrap/template/models/criterions/template_criterion.py rename to bootstrap/templates/default/project/models/criterions/template_criterion.py diff --git a/bootstrap/template/models/criterions/__init__.py b/bootstrap/templates/default/project/models/metrics/__init__.py similarity index 100% rename from bootstrap/template/models/criterions/__init__.py rename to bootstrap/templates/default/project/models/metrics/__init__.py diff --git a/bootstrap/template/models/metrics/factory.py b/bootstrap/templates/default/project/models/metrics/factory.py similarity index 100% rename from bootstrap/template/models/metrics/factory.py rename to bootstrap/templates/default/project/models/metrics/factory.py diff --git a/bootstrap/template/models/metrics/template_metric.py b/bootstrap/templates/default/project/models/metrics/template_metric.py similarity index 100% rename from bootstrap/template/models/metrics/template_metric.py rename to bootstrap/templates/default/project/models/metrics/template_metric.py diff --git a/bootstrap/template/models/metrics/__init__.py b/bootstrap/templates/default/project/models/networks/__init__.py similarity index 100% rename from bootstrap/template/models/metrics/__init__.py rename to bootstrap/templates/default/project/models/networks/__init__.py diff --git a/bootstrap/template/models/networks/factory.py b/bootstrap/templates/default/project/models/networks/factory.py similarity index 100% rename from bootstrap/template/models/networks/factory.py rename to bootstrap/templates/default/project/models/networks/factory.py diff --git a/bootstrap/template/models/networks/template_network.py b/bootstrap/templates/default/project/models/networks/template_network.py similarity index 100% rename from bootstrap/template/models/networks/template_network.py rename to bootstrap/templates/default/project/models/networks/template_network.py diff --git a/bootstrap/template/models/networks/__init__.py b/bootstrap/templates/default/project/optimizers/__init__.py similarity index 100% rename from bootstrap/template/models/networks/__init__.py rename to bootstrap/templates/default/project/optimizers/__init__.py diff --git a/bootstrap/template/optimizers/factory.py b/bootstrap/templates/default/project/optimizers/factory.py similarity index 100% rename from bootstrap/template/optimizers/factory.py rename to bootstrap/templates/default/project/optimizers/factory.py diff --git a/bootstrap/template/optimizers/myoptimizer.py b/bootstrap/templates/default/project/optimizers/myoptimizer.py similarity index 100% rename from bootstrap/template/optimizers/myoptimizer.py rename to bootstrap/templates/default/project/optimizers/myoptimizer.py diff --git a/setup.py b/setup.py index fad6fbf..6b50ea4 100644 --- a/setup.py +++ b/setup.py @@ -156,6 +156,7 @@ # package_data={ # Optional # 'sample': ['package_data.dat'], # }, + include_package_data=True, # Although 'package_data' is the preferred approach, in some case you may # need to place data files outside of your packages. See: From 2ea4e7c7643f5d8dd1a31b752d4092843d2da1fe Mon Sep 17 00:00:00 2001 From: Adrien Pouyet Date: Wed, 1 Apr 2020 23:50:48 +0200 Subject: [PATCH 4/6] fix copying template content ; remove unsued files and rename template_* -> * --- bootstrap/new.py | 4 ++-- .../project/datasets/{template_dataset.py => dataset.py} | 0 bootstrap/templates/default/project/engines/__init__.py | 0 bootstrap/templates/default/project/engines/factory.py | 0 bootstrap/templates/default/project/engines/myengine.py | 0 .../models/criterions/{template_criterion.py => criterion.py} | 0 bootstrap/templates/default/project/models/metrics/factory.py | 4 ++-- .../project/models/metrics/{template_metric.py => metric.py} | 0 .../models/networks/{template_network.py => network.py} | 0 bootstrap/templates/default/project/optimizers/__init__.py | 0 bootstrap/templates/default/project/optimizers/factory.py | 0 bootstrap/templates/default/project/optimizers/myoptimizer.py | 0 12 files changed, 4 insertions(+), 4 deletions(-) rename bootstrap/templates/default/project/datasets/{template_dataset.py => dataset.py} (100%) delete mode 100644 bootstrap/templates/default/project/engines/__init__.py delete mode 100644 bootstrap/templates/default/project/engines/factory.py delete mode 100644 bootstrap/templates/default/project/engines/myengine.py rename bootstrap/templates/default/project/models/criterions/{template_criterion.py => criterion.py} (100%) rename bootstrap/templates/default/project/models/metrics/{template_metric.py => metric.py} (100%) rename bootstrap/templates/default/project/models/networks/{template_network.py => network.py} (100%) delete mode 100644 bootstrap/templates/default/project/optimizers/__init__.py delete mode 100644 bootstrap/templates/default/project/optimizers/factory.py delete mode 100644 bootstrap/templates/default/project/optimizers/myoptimizer.py diff --git a/bootstrap/new.py b/bootstrap/new.py index ad57e4f..ba37ea4 100644 --- a/bootstrap/new.py +++ b/bootstrap/new.py @@ -14,9 +14,9 @@ def get_template_file(filename, project_name): parts = list(filename.parts) project_index = parts.index(project_name.lower()) if parts[-1] not in ["__init__.py", "factory.py"]: - parts[-1] = "template_" + parts[-1][2:] # remove "my" + parts[-1] = parts[-1].replace("my", "") template_path = "/".join(parts[project_index + 1:]) - template_path = file_dir / Path("template") / template_path + template_path = file_dir / Path("templates/default/project") / template_path return template_path diff --git a/bootstrap/templates/default/project/datasets/template_dataset.py b/bootstrap/templates/default/project/datasets/dataset.py similarity index 100% rename from bootstrap/templates/default/project/datasets/template_dataset.py rename to bootstrap/templates/default/project/datasets/dataset.py diff --git a/bootstrap/templates/default/project/engines/__init__.py b/bootstrap/templates/default/project/engines/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bootstrap/templates/default/project/engines/factory.py b/bootstrap/templates/default/project/engines/factory.py deleted file mode 100644 index e69de29..0000000 diff --git a/bootstrap/templates/default/project/engines/myengine.py b/bootstrap/templates/default/project/engines/myengine.py deleted file mode 100644 index e69de29..0000000 diff --git a/bootstrap/templates/default/project/models/criterions/template_criterion.py b/bootstrap/templates/default/project/models/criterions/criterion.py similarity index 100% rename from bootstrap/templates/default/project/models/criterions/template_criterion.py rename to bootstrap/templates/default/project/models/criterions/criterion.py diff --git a/bootstrap/templates/default/project/models/metrics/factory.py b/bootstrap/templates/default/project/models/metrics/factory.py index f78cb52..6158428 100644 --- a/bootstrap/templates/default/project/models/metrics/factory.py +++ b/bootstrap/templates/default/project/models/metrics/factory.py @@ -6,8 +6,8 @@ def factory(engine=None, mode="train"): opt = Options()['model.metric'] - if opt['name'] == '{PROJECT_NAME_LOWER}metric': - metric = {PROJECT_NAME_LOWER}metric() + if opt['name'] == '{PROJECT_NAME_LOWER}': + metric = {PROJECT_NAME_LOWER}Metric() else: raise ValueError(opt['name']) diff --git a/bootstrap/templates/default/project/models/metrics/template_metric.py b/bootstrap/templates/default/project/models/metrics/metric.py similarity index 100% rename from bootstrap/templates/default/project/models/metrics/template_metric.py rename to bootstrap/templates/default/project/models/metrics/metric.py diff --git a/bootstrap/templates/default/project/models/networks/template_network.py b/bootstrap/templates/default/project/models/networks/network.py similarity index 100% rename from bootstrap/templates/default/project/models/networks/template_network.py rename to bootstrap/templates/default/project/models/networks/network.py diff --git a/bootstrap/templates/default/project/optimizers/__init__.py b/bootstrap/templates/default/project/optimizers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bootstrap/templates/default/project/optimizers/factory.py b/bootstrap/templates/default/project/optimizers/factory.py deleted file mode 100644 index e69de29..0000000 diff --git a/bootstrap/templates/default/project/optimizers/myoptimizer.py b/bootstrap/templates/default/project/optimizers/myoptimizer.py deleted file mode 100644 index e69de29..0000000 From fdcebbb320de21b10ec5d8790d47359f647b677c Mon Sep 17 00:00:00 2001 From: Remi Cadene Date: Mon, 11 May 2020 01:16:17 +0200 Subject: [PATCH 5/6] Add working version and Tests --- bootstrap/new.py | 117 +++++--------- bootstrap/templates/default/.gitignore | 118 ++++++++++++++ bootstrap/templates/default/LICENSE.txt | 29 ++++ bootstrap/templates/default/MANIFEST.in | 2 + bootstrap/templates/default/README.md | 33 ++++ .../templates/default/project/__version__.py | 1 + .../default/project/datasets/dataset.py | 63 ++++++-- .../default/project/datasets/factory.py | 36 ++--- .../project/models/criterions/criterion.py | 13 +- .../project/models/criterions/factory.py | 14 +- .../default/project/models/metrics/factory.py | 8 +- .../default/project/models/metrics/metric.py | 19 +-- .../project/models/networks/factory.py | 20 +-- .../project/models/networks/network.py | 17 +- .../default/project/options/options.yaml | 46 ++++++ bootstrap/templates/default/requirements.txt | 1 + bootstrap/templates/default/setup.cfg | 2 + bootstrap/templates/default/setup.py | 151 ++++++++++++++++++ tests/test_new.py | 29 ++++ 19 files changed, 559 insertions(+), 160 deletions(-) create mode 100644 bootstrap/templates/default/.gitignore create mode 100644 bootstrap/templates/default/LICENSE.txt create mode 100644 bootstrap/templates/default/MANIFEST.in create mode 100644 bootstrap/templates/default/README.md create mode 100644 bootstrap/templates/default/project/__version__.py create mode 100644 bootstrap/templates/default/project/options/options.yaml create mode 100644 bootstrap/templates/default/requirements.txt create mode 100644 bootstrap/templates/default/setup.cfg create mode 100644 bootstrap/templates/default/setup.py create mode 100644 tests/test_new.py diff --git a/bootstrap/new.py b/bootstrap/new.py index ba37ea4..c511157 100644 --- a/bootstrap/new.py +++ b/bootstrap/new.py @@ -1,91 +1,54 @@ -import os from pathlib import Path from argparse import ArgumentParser - -file_dir = Path(__file__).parent - - -parser = ArgumentParser() -parser.add_argument("--project_name", type=str, help="Project name") - - -def get_template_file(filename, project_name): - parts = list(filename.parts) - project_index = parts.index(project_name.lower()) - if parts[-1] not in ["__init__.py", "factory.py"]: - parts[-1] = parts[-1].replace("my", "") - template_path = "/".join(parts[project_index + 1:]) - template_path = file_dir / Path("templates/default/project") / template_path - - return template_path - - -def get_file_content(filename, project_name): - template = get_template_file(filename, project_name) - - content = Path(template).read_text() - content = content.replace("{PROJECT_NAME}", project_name) - content = content.replace("{PROJECT_NAME_LOWER}", project_name.lower()) - content = content.replace("{PROJECT_NAME_UPPER}", project_name.upper()) - +def replace_content(file_path, prj_name): + content = file_path.read_text() + content = content.replace('{PROJECT_NAME}', prj_name) + content = content.replace('{PROJECT_NAME_LOWER}', prj_name.lower()) + #content = content.replace('{PROJECT_NAME_UPPER}', pname.upper()) return content +def new_project(prj_name, prj_dir): + # will be rename into project_name.lower() + suffix + # ex: dataset.py -> myproject.py + files_to_rename = ['dataset.py', 'criterion.py', 'metric.py', 'network.py', 'options.yaml'] -def write_files(files, project_name): - for f in files: - content = get_file_content(f, project_name) - f.write_text(content) - + # will be rename into project_name.lower() + # ex: project/datasets -> myproject/datasets + dirs_to_rename = ['project'] -def get_files(directory): - dir_name = directory.stem - if dir_name == "options": - return [directory / "abstract.yaml"] - - to_ret = [] - if dir_name != "models": - to_ret.append(directory / "__init__.py") - - to_ret.append(directory / "factory.py") - custom_file = f"my{dir_name[:-1]}.py" - to_ret.append(directory / custom_file) - - return to_ret - - -if __name__ == "__main__": - args = parser.parse_args() - project_name = args.project_name - - path = Path(f"{project_name.lower()}.bootstrap.pytorch") + path = Path(prj_dir) + path = path / f'{prj_name.lower()}.bootstrap.pytorch' path.mkdir() + tpl_path = Path(__file__).parent / 'templates' / 'default' - print(f"Creating logs directory") - os.mkdir(path / "logs") + print(f'Creating project {prj_name.lower()} in {path}') - print(f"Creating project directory and __init__.py file") - path = Path(f"{project_name.lower()}.bootstrap.pytorch/{project_name.lower()}") - path.mkdir() - Path(path / "__init__.py").touch() + # recursive iteration over directories and files + for p in tpl_path.rglob('*'): - print("Creating models directory and __init__ file") - Path(path / "models").mkdir() - Path(path / "models/__init__.py").touch() + # absolute path to local path + # ex: bootstrap.pytorch/templates/default/project -> project + tpl_local_path = p.relative_to(tpl_path) - directories = [ - "datasets", - "models/networks", - "models/criterions", - "models/metrics", - ] + # replace name of directories + local_path = p.relative_to(tpl_path) + for dir_name in dirs_to_rename: + local_path = Path(str(local_path).replace(dir_name, prj_name.lower())) - for directory in directories: - print(f"Creating {directory} folder and associated files") - new_dir = path / directory - if directory != "models": - new_dir.mkdir() - files = get_files(new_dir) - write_files(files, project_name) + if p.is_dir(): + Path(path / local_path).mkdir() - print("Project is ready !") + if p.is_file(): + content = replace_content(tpl_path / tpl_local_path, prj_name) + if p.name in files_to_rename: + local_path = Path(local_path.parent / f'{prj_name.lower()}{p.suffix}') + print(local_path) + Path(path / local_path).write_text(content) + +if __name__ == '__main__': + parser = ArgumentParser() + parser.add_argument('--project_name', type=str, default='MyProject') + parser.add_argument('--project_dir', type=str, default='.') + args = parser.parse_args() + new_project(args.project_name, args.project_dir) diff --git a/bootstrap/templates/default/.gitignore b/bootstrap/templates/default/.gitignore new file mode 100644 index 0000000..616dfb7 --- /dev/null +++ b/bootstrap/templates/default/.gitignore @@ -0,0 +1,118 @@ +# Boostrap.pytorch +logs/* +data/* +!.gitkeep +docs/src +!logger.py + +# Apple +.DS_Store +._.DS_Store + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +# lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +.static_storage/ +.media/ +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ + +*.swp +*.nfs* diff --git a/bootstrap/templates/default/LICENSE.txt b/bootstrap/templates/default/LICENSE.txt new file mode 100644 index 0000000..121c97f --- /dev/null +++ b/bootstrap/templates/default/LICENSE.txt @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2020+, {PROJECT_NAME} +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/bootstrap/templates/default/MANIFEST.in b/bootstrap/templates/default/MANIFEST.in new file mode 100644 index 0000000..cd797d8 --- /dev/null +++ b/bootstrap/templates/default/MANIFEST.in @@ -0,0 +1,2 @@ +include LICENSE.txt +include README.md \ No newline at end of file diff --git a/bootstrap/templates/default/README.md b/bootstrap/templates/default/README.md new file mode 100644 index 0000000..0c49a14 --- /dev/null +++ b/bootstrap/templates/default/README.md @@ -0,0 +1,33 @@ +# {PROJECT_NAME} + +## Install + +[Conda](https://docs.conda.io/en/latest/miniconda.html) + +```bash +conda create --name {PROJECT_NAME_LOWER} python=3 +source activate {PROJECT_NAME_LOWER} + +cd $HOME +git clone --recursive https://github.com/{PROJECT_NAME}/{PROJECT_NAME_LOWER}.bootstrap.pytorch.git +cd {PROJECT_NAME_LOWER}.bootstrap.pytorch +pip install -r requirements.txt +``` + +## Reproducing results + +Run experiment: +```bash +python -m bootstrap.run \ +-o {PROJECT_NAME_LOWER}/options/{PROJECT_NAME_LOWER}.yaml \ +--exp.dir logs/{PROJECT_NAME_LOWER}/1_exp +``` + +Display training and evaluation figures: +```bash +open logs/{PROJECT_NAME_LOWER}/1_exp/view.html +``` + +Display table of results: +```bash +python -m bootstrap.compare -o \ No newline at end of file diff --git a/bootstrap/templates/default/project/__version__.py b/bootstrap/templates/default/project/__version__.py new file mode 100644 index 0000000..c57bfd5 --- /dev/null +++ b/bootstrap/templates/default/project/__version__.py @@ -0,0 +1 @@ +__version__ = '0.0.0' diff --git a/bootstrap/templates/default/project/datasets/dataset.py b/bootstrap/templates/default/project/datasets/dataset.py index 3a3d641..b8121ee 100644 --- a/bootstrap/templates/default/project/datasets/dataset.py +++ b/bootstrap/templates/default/project/datasets/dataset.py @@ -1,23 +1,54 @@ -from bootstrap.datasets.dataset import Dataset +import torch +import torch.utils.data as tdata +from bootstrap.datasets import transforms as btf -class {PROJECT_NAME}Dataset(Dataset): - """ Dataset of Wikipedia Comparable Article +class {PROJECT_NAME}Dataset(tdata.Dataset): - Parameters - ----------- - """ def __init__(self, - dir_data, - split='train', - batch_size=4, - shuffle=False, - pin_memory=False, - nb_threads=4): - super({PROJECT_NAME}Dataset, self).__init__(dir_data, split, batch_size, shuffle, pin_memory, nb_threads) + dir_data, + split='train', + batch_size=4, + shuffle=False, + pin_memory=False, + nb_threads=4, + *args, + **kwargs): + self.dir_data = dir_data + self.split = split + self.batch_size = batch_size + self.shuffle = shuffle + self.pin_memory = pin_memory + self.nb_threads = nb_threads + self.sampler = None + + self.collate_fn = btf.Compose([ + btf.ListDictsToDictLists(), + btf.StackTensors() + ]) + + self.nb_items = kwargs['nb_items'] + self.data = torch.randn(self.nb_items, 10) + self.target = torch.zeros(self.nb_items) + self.target[:int(self.nb_items / 2)].fill_(1) + #self.target[int(self.nb_items / 2):, 0].fill_(1) + + def make_batch_loader(self, batch_size=None, shuffle=None): + batch_loader = tdata.DataLoader( + dataset=self, + batch_size=self.batch_size if batch_size is None else batch_size, + shuffle=self.shuffle if shuffle is None else shuffle, + pin_memory=self.pin_memory, + num_workers=self.nb_threads, + collate_fn=self.collate_fn, + sampler=self.sampler) + return batch_loader def __len__(self): - raise NotImplementedError + return self.data.shape[0] - def __getitem__(self, i): - raise NotImplementedError + def __getitem__(self, idx): + item = {} + item['data'] = self.data[idx] + item['target'] = self.target[idx] + return item diff --git a/bootstrap/templates/default/project/datasets/factory.py b/bootstrap/templates/default/project/datasets/factory.py index 8f86d76..8fa5afe 100644 --- a/bootstrap/templates/default/project/datasets/factory.py +++ b/bootstrap/templates/default/project/datasets/factory.py @@ -1,42 +1,40 @@ from bootstrap.lib.options import Options from bootstrap.lib.logger import Logger - -from .mydataset import {PROJECT_NAME}Dataset +from .{PROJECT_NAME_LOWER} import {PROJECT_NAME}Dataset def factory(engine=None): - logger = Logger() - logger('Creating dataset...') + Logger()('Creating dataset...') - opt = Options()["dataset"] + opt = Options()['dataset'] dataset = {} - if opt.get("train_split", None): - logger("Loading train data") - dataset["train"] = factory_split(opt["train_split"]) - logger(f"Train dataset length is {len(dataset['train'])}") + if opt.get('train_split', None): + dataset['train'] = factory_split(opt['train_split']) - if opt.get("eval_split", None): - logger("Loading test data") - dataset["eval"] = factory_split(opt["eval_split"]) - logger(f"Test dataset length is {len(dataset['eval'])}") + if opt.get('eval_split', None): + dataset['eval'] = factory_split(opt['eval_split']) - logger("Dataset was created") return dataset def factory_split(split): - opt = Options()["dataset"] + opt = Options()['dataset'] - shuffle = ("train" in split) + shuffle = ('train' in split) + dict_opt = opt.asdict() + dict_opt.pop('dir', None) + dict_opt.pop('batch_size', None) + dict_opt.pop('nb_threads', None) dataset = {PROJECT_NAME}Dataset( - dir_data=opt["dir"], + dir_data=opt['dir'], split=split, - batch_size=opt["batch_size"], + batch_size=opt['batch_size'], shuffle=shuffle, - nb_threads=opt["nb_threads"] + nb_threads=opt['nb_threads'], + **dict_opt ) return dataset diff --git a/bootstrap/templates/default/project/models/criterions/criterion.py b/bootstrap/templates/default/project/models/criterions/criterion.py index e331f35..59654ca 100644 --- a/bootstrap/templates/default/project/models/criterions/criterion.py +++ b/bootstrap/templates/default/project/models/criterions/criterion.py @@ -1,12 +1,15 @@ +import torch import torch.nn as nn - class {PROJECT_NAME}Criterion(nn.Module): - def __init__(self): + def __init__(self, *args, **kwargs): super({PROJECT_NAME}Criterion, self).__init__() + self.bce_loss = nn.BCELoss() def forward(self, net_out, batch): - # net_out : output of network - # batch : output of dataset (after collate function) - raise NotImplementedError + pred = net_out['pred'].squeeze(1) + target = batch['target'] + loss = self.bce_loss(pred, target) + out = {'loss': loss} + return out diff --git a/bootstrap/templates/default/project/models/criterions/factory.py b/bootstrap/templates/default/project/models/criterions/factory.py index f4560b9..27af30f 100644 --- a/bootstrap/templates/default/project/models/criterions/factory.py +++ b/bootstrap/templates/default/project/models/criterions/factory.py @@ -1,16 +1,12 @@ from bootstrap.lib.options import Options -from bootstrap.lib.logger import Logger - -from .mycriterion import {PROJECT_NAME}Criterion - +from .{PROJECT_NAME_LOWER} import {PROJECT_NAME}Criterion def factory(engine=None, mode=None): - logger = Logger() - logger('Creating criterion for {} mode...'.format(mode)) + opt = Options()['model.criterion'] - if Options()['model']['criterion'].get('import', False): - criterion = {PROJECT_NAME}Criterion() + if opt['name'] == '{PROJECT_NAME_LOWER}': + criterion = {PROJECT_NAME}Criterion(**opt) else: - raise ValueError() + raise ValueError(opt['name']) return criterion diff --git a/bootstrap/templates/default/project/models/metrics/factory.py b/bootstrap/templates/default/project/models/metrics/factory.py index 6158428..e6299d7 100644 --- a/bootstrap/templates/default/project/models/metrics/factory.py +++ b/bootstrap/templates/default/project/models/metrics/factory.py @@ -1,13 +1,11 @@ from bootstrap.lib.options import Options +from .{PROJECT_NAME_LOWER} import {PROJECT_NAME}Metric -from .mymetric import {PROJECT_NAME}Metric - - -def factory(engine=None, mode="train"): +def factory(engine=None, mode='train'): opt = Options()['model.metric'] if opt['name'] == '{PROJECT_NAME_LOWER}': - metric = {PROJECT_NAME_LOWER}Metric() + metric = {PROJECT_NAME}Metric(**opt) else: raise ValueError(opt['name']) diff --git a/bootstrap/templates/default/project/models/metrics/metric.py b/bootstrap/templates/default/project/models/metrics/metric.py index b6fd58a..086c4ed 100644 --- a/bootstrap/templates/default/project/models/metrics/metric.py +++ b/bootstrap/templates/default/project/models/metrics/metric.py @@ -1,13 +1,14 @@ import torch.nn as nn -from bootstrap.lib.logger import Logger - class {PROJECT_NAME}Metric(nn.Module): - def __init__(self): - super(Accuracy, self).__init__() - def forward(self, crit_out, net_out, batch): - # crit_out : output of criterion (dictionnary) - # net_out : output of network - # batch : output of dataset (after collate function) - raise NotImplementedError + def __init__(self, *args, **kwargs): + super({PROJECT_NAME}Metric, self).__init__() + self.thresh = kwargs['thresh'] + + def forward(self, cri_out, net_out, batch): + pred = net_out['pred'] > self.thresh + target = batch['target'] + acc = (pred == target).float().mean() + out = {'accuracy': acc} + return out diff --git a/bootstrap/templates/default/project/models/networks/factory.py b/bootstrap/templates/default/project/models/networks/factory.py index fa7e463..9794a75 100644 --- a/bootstrap/templates/default/project/models/networks/factory.py +++ b/bootstrap/templates/default/project/models/networks/factory.py @@ -1,22 +1,16 @@ +import torch from bootstrap.lib.options import Options -from bootstrap.lib.logger import Logger from bootstrap.models.networks.data_parallel import DataParallel - -from .mynetwork import {PROJECT_NAME}Network - +from .{PROJECT_NAME_LOWER} import {PROJECT_NAME}Network def factory(engine): - logger = Logger() - net_opt = Options()["model"]["network"] - logger("Creating Network...") + opt = Options()['model.network'] - if net_opt["name"] == "{PROJECT_NAME_LOWER}network": - # You can use any param to create your network - # You just have to write them in your option file from options/ folder - net = {PROJECT_NAME}Network(net_opt["param1"], net_opt["param2"]) + if opt['name'] == '{PROJECT_NAME_LOWER}': + net = {PROJECT_NAME}Network(**opt) else: - raise ValueError(opt["name"]) - logger("Network was created") + raise ValueError(opt['name']) + if torch.cuda.device_count() > 1: net = DataParallel(net) return net diff --git a/bootstrap/templates/default/project/models/networks/network.py b/bootstrap/templates/default/project/models/networks/network.py index f10c7b1..e88e595 100644 --- a/bootstrap/templates/default/project/models/networks/network.py +++ b/bootstrap/templates/default/project/models/networks/network.py @@ -1,12 +1,15 @@ import torch.nn as nn - class {PROJECT_NAME}Network(nn.Module): + def __init__(self, *args, **kwargs): - super(MyNetwork, self).__init__() - # Assign args + super({PROJECT_NAME}Network, self).__init__() + self.net = nn.Sequential( + nn.Linear(kwargs['dim_in'], kwargs['dim_out']), + nn.Sigmoid()) - def forward(self, x): - # x is a dictionnary given by Dataset class - pred = self.net(x) - return pred # This is a tensor (or several tensors) \ No newline at end of file + def forward(self, batch): + x = batch['data'] + y = self.net(x) + out = {'pred': y} + return out diff --git a/bootstrap/templates/default/project/options/options.yaml b/bootstrap/templates/default/project/options/options.yaml new file mode 100644 index 0000000..bf7f6ec --- /dev/null +++ b/bootstrap/templates/default/project/options/options.yaml @@ -0,0 +1,46 @@ +exp: + dir: logs/{PROJECT_NAME_LOWER}/1_exp + resume: # last, best_[...], or empty (from scratch) +dataset: + import: {PROJECT_NAME_LOWER}.datasets.factory + name: {PROJECT_NAME_LOWER} + dir: data/{PROJECT_NAME_LOWER} + train_split: train + eval_split: val + nb_threads: 4 + batch_size: 64 + nb_items: 100 +model: + name: default + network: + import: {PROJECT_NAME_LOWER}.models.networks.factory + name: {PROJECT_NAME_LOWER} + dim_in: 10 + dim_out: 1 + criterion: + import: {PROJECT_NAME_LOWER}.models.criterions.factory + name: {PROJECT_NAME_LOWER} + metric: + import: {PROJECT_NAME_LOWER}.models.metrics.factory + name: {PROJECT_NAME_LOWER} + thresh: 0.5 +optimizer: + name: adam + lr: 0.0004 +engine: + name: default + debug: False + nb_epochs: 10 + print_freq: 10 + saving_criteria: + - loss:min # save when new_best < best + - accuracy:max # save when new_best > best +misc: + cuda: True + seed: 1337 +views: + name: plotly + items: + - logs:train_epoch.loss+logs:eval_epoch.loss + - logs:train_batch.loss + - logs:train_epoch.accuracy+logs:eval_epoch.accuracy diff --git a/bootstrap/templates/default/requirements.txt b/bootstrap/templates/default/requirements.txt new file mode 100644 index 0000000..4446a1f --- /dev/null +++ b/bootstrap/templates/default/requirements.txt @@ -0,0 +1 @@ +bootstrap.pytorch \ No newline at end of file diff --git a/bootstrap/templates/default/setup.cfg b/bootstrap/templates/default/setup.cfg new file mode 100644 index 0000000..224a779 --- /dev/null +++ b/bootstrap/templates/default/setup.cfg @@ -0,0 +1,2 @@ +[metadata] +description-file = README.md \ No newline at end of file diff --git a/bootstrap/templates/default/setup.py b/bootstrap/templates/default/setup.py new file mode 100644 index 0000000..9fa44fd --- /dev/null +++ b/bootstrap/templates/default/setup.py @@ -0,0 +1,151 @@ +"""A setuptools based setup module. + +See: +https://packaging.python.org/en/latest/distributing.html +https://github.com/pypa/sampleproject +""" + +# Always prefer setuptools over distutils +from setuptools import setup, find_packages +# To use a consistent encoding +from codecs import open +from os import path + +here = path.abspath(path.dirname(__file__)) + +# Get the long description from the README file +with open(path.join(here, 'README.md'), encoding='utf-8') as f: + long_description = f.read() + +# Arguments marked as "Required" below must be included for upload to PyPI. +# Fields marked as "Optional" may be commented out. + +# https://stackoverflow.com/questions/458550/standard-way-to-embed-version-into-python-package/16084844#16084844 +exec(open(path.join(here, '{PROJECT_NAME_LOWER}', '__version__.py')).read()) +setup( + # This is the name of your project. The first time you publish this + # package, this name will be registered for you. It will determine how + # users can install this project, e.g.: + # + # $ pip install sampleproject + # + # And where it will live on PyPI: https://pypi.org/project/sampleproject/ + # + # There are some restrictions on what makes a valid project name + # specification here: + # https://packaging.python.org/specifications/core-metadata/#name + name='{PROJECT_NAME_LOWER}.bootstrap.pytorch', # Required + + # Versions should comply with PEP 440: + # https://www.python.org/dev/peps/pep-0440/ + # + # For a discussion on single-sourcing the version across setup.py and the + # project code, see + # https://packaging.python.org/en/latest/single_source_version.html + version=__version__, # Required + + # This is a one-line description or tagline of what your project does. This + # corresponds to the "Summary" metadata field: + # https://packaging.python.org/specifications/core-metadata/#summary + description='{PROJECT_NAME}', # Required + + # This is an optional longer description of your project that represents + # the body of text which users will see when they visit PyPI. + # + # Often, this is the same as your README, so you can just read it in from + # that file directly (as we have already done above) + # + # This field corresponds to the "Description" metadata field: + # https://packaging.python.org/specifications/core-metadata/#description-optional + long_description=long_description, # Optional + + # This should be a valid link to your project's main homepage. + # + # This field corresponds to the "Home-Page" metadata field: + # https://packaging.python.org/specifications/core-metadata/#home-page-optional + url='https://github.com/{PROJECT_NAME}/{PROJECT_NAME_LOWER}.bootstrap.pytorch', # Optional + + # This should be your name or the name of the organization which owns the + # project. + author='{PROJECT_NAME}', # Optional + + # This should be a valid email address corresponding to the author listed + # above. + author_email='contact@{PROJECT_NAME_LOWER}.com', # Optional + + # Classifiers help users find your project by categorizing it. + # + # For a list of valid classifiers, see + # https://pypi.python.org/pypi?%3Aaction=list_classifiers + classifiers=[ # Optional + # How mature is this project? Common values are + # 3 - Alpha + # 4 - Beta + # 5 - Production/Stable + 'Development Status :: 3 - Alpha', + + # Indicate who your project is intended for + 'Intended Audience :: Developers', + 'Topic :: Software Development :: Build Tools', + + # Pick your license as you wish + 'License :: OSI Approved :: MIT License', + + # Specify the Python versions you support here. In particular, ensure + # that you indicate whether you support Python 2, Python 3 or both. + 'Programming Language :: Python :: 3.7', + ], + + # This field adds keywords for your project which will appear on the + # project page. What does your project relate to? + # + # Note that this is a string of words separated by whitespace, not a list. + keywords='pytorch framework bootstrap deep learning scaffolding', # Optional + + # You can just specify package directories manually here if your project is + # simple. Or you can use find_packages(). + # + # Alternatively, if you just want to distribute a single Python file, use + # the `py_modules` argument instead as follows, which will expect a file + # called `my_module.py` to exist: + # + # py_modules=["my_module"], + # + packages=find_packages(exclude=[ + 'data', + 'logs', + ]), + + # This field lists other packages that your project depends on to run. + # Any package you put here will be installed by pip when your project is + # installed, so they must be valid existing projects. + # + # For an analysis of "install_requires" vs pip's requirements files see: + # https://packaging.python.org/en/latest/requirements.html + install_requires=[ + 'bootstrap.pytorch' + ], + + # List additional groups of dependencies here (e.g. development + # dependencies). Users will be able to install these using the "extras" + # syntax, for example: + # + # $ pip install sampleproject[dev] + # + # Similar to `install_requires` above, these must be valid existing + # projects. + # extras_require={ # Optional + # 'dev': ['check-manifest'], + # 'test': ['coverage'], + # }, + + # If there are data files included in your packages that need to be + # installed, specify them here. + # + # If using Python 2.6 or earlier, then these have to be included in + # MANIFEST.in as well. + # package_data={ # Optional + # 'sample': ['package_data.dat'], + # }, + include_package_data=True, +) diff --git a/tests/test_new.py b/tests/test_new.py new file mode 100644 index 0000000..851a808 --- /dev/null +++ b/tests/test_new.py @@ -0,0 +1,29 @@ +import os +import pytest + +def test_new(tmpdir): + exit_status = os.system(f'python -m bootstrap.new --project_name MyProject --project_dir {tmpdir}') + assert exit_status == 0 + + os.chdir(os.path.join(tmpdir, 'myproject.bootstrap.pytorch')) + exit_status = os.system('python -m bootstrap.run -o myproject/options/myproject.yaml --exp.dir logs/myproject/1_exp --misc.cuda False --engine.nb_epochs 10') + assert exit_status == 0 + + fnames = [ + 'ckpt_best_accuracy_engine.pth.tar', + 'ckpt_best_loss_optimizer.pth.tar', + 'logs.txt', + 'ckpt_best_accuracy_model.pth.tar', + 'ckpt_last_engine.pth.tar', + 'options.yaml', + 'ckpt_best_accuracy_optimizer.pth.tar', + 'ckpt_last_model.pth.tar', + 'view.html', + 'ckpt_best_loss_engine.pth.tar', + 'ckpt_last_optimizer.pth.tar', + 'ckpt_best_loss_model.pth.tar', + 'logs.json' + ] + + for fname in fnames: + assert os.path.isfile(f'logs/myproject/1_exp/{fname}') From 4cbe7978b237ce0839e674572c8919b1e70fff86 Mon Sep 17 00:00:00 2001 From: Remi Cadene Date: Mon, 11 May 2020 01:42:09 +0200 Subject: [PATCH 6/6] Format to flake8 --- bootstrap/new.py | 5 ++++- bootstrap/templates/default/project/datasets/dataset.py | 6 +++--- bootstrap/templates/default/project/datasets/factory.py | 4 ++-- .../default/project/models/criterions/criterion.py | 5 +++-- .../templates/default/project/models/criterions/factory.py | 5 +++-- .../templates/default/project/models/metrics/factory.py | 5 +++-- .../templates/default/project/models/metrics/metric.py | 5 +++-- .../templates/default/project/models/networks/factory.py | 5 +++-- .../templates/default/project/models/networks/network.py | 5 +++-- bootstrap/templates/default/setup.py | 6 +++--- 10 files changed, 30 insertions(+), 21 deletions(-) diff --git a/bootstrap/new.py b/bootstrap/new.py index c511157..c57b453 100644 --- a/bootstrap/new.py +++ b/bootstrap/new.py @@ -1,13 +1,15 @@ from pathlib import Path from argparse import ArgumentParser + def replace_content(file_path, prj_name): content = file_path.read_text() content = content.replace('{PROJECT_NAME}', prj_name) content = content.replace('{PROJECT_NAME_LOWER}', prj_name.lower()) - #content = content.replace('{PROJECT_NAME_UPPER}', pname.upper()) + content = content.replace(' # noqa: E999', '') return content + def new_project(prj_name, prj_dir): # will be rename into project_name.lower() + suffix # ex: dataset.py -> myproject.py @@ -46,6 +48,7 @@ def new_project(prj_name, prj_dir): print(local_path) Path(path / local_path).write_text(content) + if __name__ == '__main__': parser = ArgumentParser() parser.add_argument('--project_name', type=str, default='MyProject') diff --git a/bootstrap/templates/default/project/datasets/dataset.py b/bootstrap/templates/default/project/datasets/dataset.py index b8121ee..142f8a1 100644 --- a/bootstrap/templates/default/project/datasets/dataset.py +++ b/bootstrap/templates/default/project/datasets/dataset.py @@ -3,9 +3,10 @@ from bootstrap.datasets import transforms as btf -class {PROJECT_NAME}Dataset(tdata.Dataset): +class {PROJECT_NAME}Dataset(tdata.Dataset): # noqa: E999 - def __init__(self, + def __init__( + self, dir_data, split='train', batch_size=4, @@ -31,7 +32,6 @@ def __init__(self, self.data = torch.randn(self.nb_items, 10) self.target = torch.zeros(self.nb_items) self.target[:int(self.nb_items / 2)].fill_(1) - #self.target[int(self.nb_items / 2):, 0].fill_(1) def make_batch_loader(self, batch_size=None, shuffle=None): batch_loader = tdata.DataLoader( diff --git a/bootstrap/templates/default/project/datasets/factory.py b/bootstrap/templates/default/project/datasets/factory.py index 8fa5afe..5ece23d 100644 --- a/bootstrap/templates/default/project/datasets/factory.py +++ b/bootstrap/templates/default/project/datasets/factory.py @@ -1,6 +1,6 @@ from bootstrap.lib.options import Options from bootstrap.lib.logger import Logger -from .{PROJECT_NAME_LOWER} import {PROJECT_NAME}Dataset +from .{PROJECT_NAME_LOWER} import {PROJECT_NAME}Dataset # noqa: E999 def factory(engine=None): @@ -28,7 +28,7 @@ def factory_split(split): dict_opt.pop('dir', None) dict_opt.pop('batch_size', None) dict_opt.pop('nb_threads', None) - dataset = {PROJECT_NAME}Dataset( + dataset = {PROJECT_NAME}Dataset( # noqa: E999 dir_data=opt['dir'], split=split, batch_size=opt['batch_size'], diff --git a/bootstrap/templates/default/project/models/criterions/criterion.py b/bootstrap/templates/default/project/models/criterions/criterion.py index 59654ca..5630b4e 100644 --- a/bootstrap/templates/default/project/models/criterions/criterion.py +++ b/bootstrap/templates/default/project/models/criterions/criterion.py @@ -1,10 +1,11 @@ import torch import torch.nn as nn -class {PROJECT_NAME}Criterion(nn.Module): + +class {PROJECT_NAME}Criterion(nn.Module): # noqa: E999 def __init__(self, *args, **kwargs): - super({PROJECT_NAME}Criterion, self).__init__() + super({PROJECT_NAME}Criterion, self).__init__() # noqa: E999 self.bce_loss = nn.BCELoss() def forward(self, net_out, batch): diff --git a/bootstrap/templates/default/project/models/criterions/factory.py b/bootstrap/templates/default/project/models/criterions/factory.py index 27af30f..538623d 100644 --- a/bootstrap/templates/default/project/models/criterions/factory.py +++ b/bootstrap/templates/default/project/models/criterions/factory.py @@ -1,11 +1,12 @@ from bootstrap.lib.options import Options -from .{PROJECT_NAME_LOWER} import {PROJECT_NAME}Criterion +from .{PROJECT_NAME_LOWER} import {PROJECT_NAME}Criterion # noqa: E999 + def factory(engine=None, mode=None): opt = Options()['model.criterion'] if opt['name'] == '{PROJECT_NAME_LOWER}': - criterion = {PROJECT_NAME}Criterion(**opt) + criterion = {PROJECT_NAME}Criterion(**opt) # noqa: E999 else: raise ValueError(opt['name']) diff --git a/bootstrap/templates/default/project/models/metrics/factory.py b/bootstrap/templates/default/project/models/metrics/factory.py index e6299d7..36f91bb 100644 --- a/bootstrap/templates/default/project/models/metrics/factory.py +++ b/bootstrap/templates/default/project/models/metrics/factory.py @@ -1,11 +1,12 @@ from bootstrap.lib.options import Options -from .{PROJECT_NAME_LOWER} import {PROJECT_NAME}Metric +from .{PROJECT_NAME_LOWER} import {PROJECT_NAME}Metric # noqa: E999 + def factory(engine=None, mode='train'): opt = Options()['model.metric'] if opt['name'] == '{PROJECT_NAME_LOWER}': - metric = {PROJECT_NAME}Metric(**opt) + metric = {PROJECT_NAME}Metric(**opt) # noqa: E999 else: raise ValueError(opt['name']) diff --git a/bootstrap/templates/default/project/models/metrics/metric.py b/bootstrap/templates/default/project/models/metrics/metric.py index 086c4ed..3c8147b 100644 --- a/bootstrap/templates/default/project/models/metrics/metric.py +++ b/bootstrap/templates/default/project/models/metrics/metric.py @@ -1,9 +1,10 @@ import torch.nn as nn -class {PROJECT_NAME}Metric(nn.Module): + +class {PROJECT_NAME}Metric(nn.Module): # noqa: E999 def __init__(self, *args, **kwargs): - super({PROJECT_NAME}Metric, self).__init__() + super({PROJECT_NAME}Metric, self).__init__() # noqa: E999 self.thresh = kwargs['thresh'] def forward(self, cri_out, net_out, batch): diff --git a/bootstrap/templates/default/project/models/networks/factory.py b/bootstrap/templates/default/project/models/networks/factory.py index 9794a75..e1d45b9 100644 --- a/bootstrap/templates/default/project/models/networks/factory.py +++ b/bootstrap/templates/default/project/models/networks/factory.py @@ -1,13 +1,14 @@ import torch from bootstrap.lib.options import Options from bootstrap.models.networks.data_parallel import DataParallel -from .{PROJECT_NAME_LOWER} import {PROJECT_NAME}Network +from .{PROJECT_NAME_LOWER} import {PROJECT_NAME}Network # noqa: E999 + def factory(engine): opt = Options()['model.network'] if opt['name'] == '{PROJECT_NAME_LOWER}': - net = {PROJECT_NAME}Network(**opt) + net = {PROJECT_NAME}Network(**opt) # noqa: E999 else: raise ValueError(opt['name']) diff --git a/bootstrap/templates/default/project/models/networks/network.py b/bootstrap/templates/default/project/models/networks/network.py index e88e595..f3653dc 100644 --- a/bootstrap/templates/default/project/models/networks/network.py +++ b/bootstrap/templates/default/project/models/networks/network.py @@ -1,9 +1,10 @@ import torch.nn as nn -class {PROJECT_NAME}Network(nn.Module): + +class {PROJECT_NAME}Network(nn.Module): # noqa: E999 def __init__(self, *args, **kwargs): - super({PROJECT_NAME}Network, self).__init__() + super({PROJECT_NAME}Network, self).__init__() # noqa: E999 self.net = nn.Sequential( nn.Linear(kwargs['dim_in'], kwargs['dim_out']), nn.Sigmoid()) diff --git a/bootstrap/templates/default/setup.py b/bootstrap/templates/default/setup.py index 9fa44fd..1718493 100644 --- a/bootstrap/templates/default/setup.py +++ b/bootstrap/templates/default/setup.py @@ -42,7 +42,7 @@ # For a discussion on single-sourcing the version across setup.py and the # project code, see # https://packaging.python.org/en/latest/single_source_version.html - version=__version__, # Required + version=__version__, # noqa: F821 # Required # This is a one-line description or tagline of what your project does. This # corresponds to the "Summary" metadata field: @@ -63,7 +63,7 @@ # # This field corresponds to the "Home-Page" metadata field: # https://packaging.python.org/specifications/core-metadata/#home-page-optional - url='https://github.com/{PROJECT_NAME}/{PROJECT_NAME_LOWER}.bootstrap.pytorch', # Optional + url='https://github.com/{PROJECT_NAME}/{PROJECT_NAME_LOWER}.bootstrap.pytorch', # noqa: E501 # Optional # This should be your name or the name of the organization which owns the # project. @@ -100,7 +100,7 @@ # project page. What does your project relate to? # # Note that this is a string of words separated by whitespace, not a list. - keywords='pytorch framework bootstrap deep learning scaffolding', # Optional + keywords='pytorch framework bootstrap deep learning scaffolding', # noqa: E501 # Optional # You can just specify package directories manually here if your project is # simple. Or you can use find_packages().