Merge pull request #1 from fmigneault/tag-0.3.1

tag 0.3.1
plstcharles · Jun 17, 2019 · 35b814a · 35b814a
2 parents 0af302f + f337e36
commit 35b814a
Show file tree

Hide file tree

Showing 55 changed files with 2,602 additions and 1,456 deletions.
diff --git a/.bumpversion.cfg b/.bumpversion.cfg
@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.8
+current_version = 0.3.1
 commit = True
 tag = True
 

diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -2,6 +2,17 @@
 Changelog
 =========
 
+0.3.0 (2019/06/12)
+--------------------------
+
+* Added dockerfile for containerized builds
+* Added object detection task & trainer implementations
+* Added CLI model/checkpoint export support
+* Added CLI dataset splitting/HDF5 support
+* Added baseline superresolution implementations
+* Added lots of new unit tests & docstrings
+* Cleaned up transform & display operations
+
 0.2.8 (2019/03/17)
 --------------------------
 

diff --git a/Dockerfile b/Dockerfile
@@ -1,8 +1,8 @@
 FROM nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04
 
-ARG PYTHON_VERSION=3.6
-ARG PYTORCH_VERSION=1.0.1
-ARG TORCHVISION_VERSION=0.2.2
+ARG PYTHON_VERSION=3.7
+ARG PYTORCH_VERSION=1.1.0
+ARG TORCHVISION_VERSION=0.3.0
 ARG MKL_VERSION=2019.1
 
 RUN apt-get update && apt-get install -y --no-install-recommends \

diff --git a/README.rst b/README.rst
@@ -24,9 +24,9 @@ Overview
     :alt: Travis-CI Build Status
     :target: https://travis-ci.org/plstcharles/thelper
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/plstcharles/thelper/v0.2.8.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/plstcharles/thelper/v0.3.1.svg
     :alt: Commits since latest release
-    :target: https://github.com/plstcharles/thelper/compare/v0.2.8...master
+    :target: https://github.com/plstcharles/thelper/compare/v0.3.1...master
 
 .. |readthedocs| image:: https://readthedocs.org/projects/thelper/badge/?version=latest
     :alt: Documentation Build Status

diff --git a/ci/meta.yaml b/ci/meta.yaml
@@ -1,5 +1,5 @@
 {% set name = "thelper" %}
-{% set version = "0.2.8" %}
+{% set version = "0.3.1" %}
 
 package:
   name: "{{ name|lower }}"

diff --git a/configs/example-classif.json b/configs/example-classif.json
@@ -138,6 +138,8 @@
         "save_freq": 1,
         "monitor": "accuracy",
         "use_tbx": true,
+        "display_train_predictions": false,
+        "callback_kwargs": {},
         "optimization": {
             "loss": {
                 "type": "torch.nn.CrossEntropyLoss",

diff --git a/configs/example-detect-minimalist.json b/configs/example-detect-minimalist.json
@@ -0,0 +1,52 @@
+{
+    "name": "example-detect-minimalist",
+    "datasets": {
+        "pascalvoc": {
+            "type": "thelper.data.PASCALVOC",
+            "params": {
+                "root": "data/pascalvoc",
+                "task": "detect",
+                "subset": "trainval",
+                "download": true,
+                "preload": false,
+                "target_labels": ["person"]
+            }
+        }
+    },
+    "loaders": {
+        "batch_size": 1,
+        "base_transforms": [
+            {
+                "operation": "torchvision.transforms.ToTensor",
+                "target_key": "image"
+            }
+        ],
+        "train_split": {
+            "pascalvoc": 0.9
+        },
+        "valid_split": {
+            "pascalvoc": 0.1
+        }
+    },
+    "model": {
+        "type" : "torchvision.models.detection.fasterrcnn_resnet50_fpn",
+        "params": {"pretrained": true}
+    },
+    "trainer": {
+        "epochs": 100,
+        "save_freq": 10,
+        "optimization": {
+            "optimizer": {
+                "type": "torch.optim.SGD",
+                "params": {
+                    "lr": 0.005,
+                    "momentum": 0.9,
+                    "weight_decay": 0.0005
+                }
+            }
+        },
+        "metrics": {
+
+        }
+    }
+}
diff --git a/configs/example-detect-viz.json b/configs/example-detect-viz.json
@@ -0,0 +1,20 @@
+{
+    "name": "example-detect-viz",
+    "datasets": {
+        "pascalvoc": {
+            "type": "thelper.data.PASCALVOC",
+            "params": {
+                "root": "data/pascalvoc",
+                "task": "detect",
+                "subset": "trainval",
+                "download": true,
+                "preload": false
+            }
+        }
+    },
+    "viz": {
+        "kwargs": {
+            "block": false
+        }
+    }
+}
diff --git a/configs/example-segm.json b/configs/example-segm.json
@@ -78,7 +78,8 @@
         "save_freq": 5,
         "monitor": "accuracy",
         "use_tbx": true,
-        "display_preds": true,
+        "display_train_predictions": false,
+        "callback_kwargs": {},
         "optimization": {
             "loss": {
                 "type": "torch.nn.CrossEntropyLoss",

diff --git a/configs/example-sr-srcnn.json b/configs/example-sr-srcnn.json
@@ -62,6 +62,8 @@
         "save_freq": 1,
         "monitor": "psnr",
         "use_tbx": true,
+        "display_train_predictions": false,
+        "callback_kwargs": {},
         "optimization": {
             "loss": {
                 "type": "torch.nn.MSELoss",

diff --git a/configs/example-sr-vdsr.json b/configs/example-sr-vdsr.json
@@ -55,6 +55,8 @@
         "save_freq": 1,
         "monitor": "psnr",
         "use_tbx": true,
+        "display_train_predictions": true,
+        "callback_kwargs": {},
         "optimization": {
             "loss": {
                 "type": "torch.nn.MSELoss",

diff --git a/docs/src/conf.py b/docs/src/conf.py
@@ -22,11 +22,24 @@
 project = 'thelper'
 copyright = '2018, Pierre-Luc St-Charles'
 author = 'Pierre-Luc St-Charles'
-version = release = '0.2.8'
+version = release = '0.3.1'
 
 
 # -- General configuration ---------------------------------------------------
 
+nitpicky = False  # caused too many dummy warnings as of 2019/05 due to attribs
+nitpick_ignore = [
+    ("py:class", "object"),
+    ("py:class", "abc.ABC"),
+    ("py:class", "torch.optim.lr_scheduler._LRScheduler"),
+    ("py:class", "torch.utils.data.dataset.Dataset"),
+    ("py:class", "torch.utils.data.dataloader.DataLoader"),
+    ("py:class", "torch.utils.data.sampler.Sampler"),
+    ("py:class", "torch.nn.modules.module.Module"),
+    ("py:class", "torch.nn.Module"),
+    ("py:class", "torchvision.transforms.transforms.Compose")
+]
+
 # If your documentation needs a minimal Sphinx version, state it here.
 #
 # needs_sphinx = '1.0'
@@ -246,6 +259,7 @@ def skip(app, what, name, obj, skip, options):
         return False
     return skip
 
+
 def run_apidoc(_):
     if on_rtd:
         argv = ["-M", "-o", ".", "../../thelper"]
@@ -261,6 +275,7 @@ def run_apidoc(_):
         argv.insert(0, apidoc.__file__)
         apidoc.main(argv)
 
+
 def setup(app):
     app.connect("autodoc-skip-member", skip)
     app.connect("builder-inited", run_apidoc)
diff --git a/setup.py b/setup.py
@@ -54,7 +54,7 @@ def _unavailable(self, e):
 
 setuptools.setup(
     name="thelper",
-    version="0.2.8",
+    version="0.3.1",
     license="Apache Software License 2.0",
     description="Training framework & tools for PyTorch-based machine learning projects.",
     long_description="%s\n%s" % (

diff --git a/tests/data/test_loaders.py b/tests/data/test_loaders.py
@@ -107,9 +107,6 @@ def __init__(self, nb_samples, nb_classes, subset, transforms=None, deepcopy=Fal
     def __getitem__(self, idx):
         return self.samples[idx]
 
-    def get_task(self):
-        return self.task
-
 
 @pytest.fixture
 def class_split_config():
@@ -139,9 +136,9 @@ def class_split_config():
 
 def test_classif_split(class_split_config):
     task, train_loader, valid_loader, test_loader = thelper.data.create_loaders(class_split_config)
-    assert task.check_compat(class_split_config["datasets"]["dataset_A"].get_task(), exact=True)
-    assert task.check_compat(class_split_config["datasets"]["dataset_B"].get_task(), exact=True)
-    assert task.check_compat(class_split_config["datasets"]["dataset_C"].get_task(), exact=True)
+    assert task.check_compat(class_split_config["datasets"]["dataset_A"].task, exact=True)
+    assert task.check_compat(class_split_config["datasets"]["dataset_B"].task, exact=True)
+    assert task.check_compat(class_split_config["datasets"]["dataset_C"].task, exact=True)
     train_samples, valid_samples, test_samples = {}, {}, {}
     train_class_counts, valid_class_counts, test_class_counts = [0] * 10, [0] * 10, [0] * 10
     for loader, samples, class_counts in [(train_loader, train_samples, train_class_counts),
@@ -165,9 +162,9 @@ def test_classif_split(class_split_config):
 def test_classif_split_no_balancing(class_split_config):
     class_split_config["loaders"]["skip_class_balancing"] = True
     task, train_loader, valid_loader, test_loader = thelper.data.create_loaders(class_split_config)
-    assert task.check_compat(class_split_config["datasets"]["dataset_A"].get_task(), exact=True)
-    assert task.check_compat(class_split_config["datasets"]["dataset_B"].get_task(), exact=True)
-    assert task.check_compat(class_split_config["datasets"]["dataset_C"].get_task(), exact=True)
+    assert task.check_compat(class_split_config["datasets"]["dataset_A"].task, exact=True)
+    assert task.check_compat(class_split_config["datasets"]["dataset_B"].task, exact=True)
+    assert task.check_compat(class_split_config["datasets"]["dataset_C"].task, exact=True)
     train_samples, valid_samples, test_samples = {}, {}, {}
     for loader, samples in [(train_loader, train_samples), (valid_loader, valid_samples), (test_loader, test_samples)]:
         for batch in loader:
@@ -189,20 +186,20 @@ def test_loader_factory(class_split_config, mocker):
     fake_loaders_config = copy.deepcopy(class_split_config["loaders"])
     fake_loaders_config["train_batch_size"] = 16
     with pytest.raises(AssertionError):
-        _ = thelper.data.loaders._LoaderFactory(fake_loaders_config)
-    factory = thelper.data.loaders._LoaderFactory(class_split_config["loaders"])
+        _ = thelper.data.loaders.LoaderFactory(fake_loaders_config)
+    factory = thelper.data.loaders.LoaderFactory(class_split_config["loaders"])
     assert factory.train_split["dataset_A"] == 0.5 and factory.valid_split["dataset_A"] == 0.4 and factory.test_split["dataset_A"] == 0.1
     assert factory.train_split["dataset_B"] == 0.7 and factory.valid_split["dataset_B"] == 0.3
     assert factory.test_split["dataset_C"] == 1.0
     loaders_config_drop = copy.deepcopy(class_split_config["loaders"])
     loaders_config_drop["drop_last"] = True
-    factory = thelper.data.loaders._LoaderFactory(loaders_config_drop)
+    factory = thelper.data.loaders.LoaderFactory(loaders_config_drop)
     assert factory.drop_last
     loaders_config_sampler = copy.deepcopy(class_split_config["loaders"])
     loaders_config_sampler["sampler"] = {
         "type": "torch.utils.data.sampler.RandomSampler"
     }
-    factory = thelper.data.loaders._LoaderFactory(loaders_config_sampler)
+    factory = thelper.data.loaders.LoaderFactory(loaders_config_sampler)
     assert factory.sampler_type == torch.utils.data.sampler.RandomSampler
     assert factory.train_sampler
     _ = mocker.patch("thelper.transforms.load_transforms", return_value="dummy")
@@ -221,7 +218,7 @@ def test_loader_factory(class_split_config, mocker):
             }
         }
     ]
-    factory = thelper.data.loaders._LoaderFactory(loaders_config_transfs)
+    factory = thelper.data.loaders.LoaderFactory(loaders_config_transfs)
     assert factory.base_transforms is not None
     assert factory.get_base_transforms() == factory.base_transforms
     loaders_config_transfs["train_augments"] = {"append": True, "transforms": [
@@ -238,51 +235,51 @@ def test_loader_factory(class_split_config, mocker):
             }
         }
     ]}
-    factory = thelper.data.loaders._LoaderFactory(loaders_config_transfs)
+    factory = thelper.data.loaders.LoaderFactory(loaders_config_transfs)
     assert factory.train_augments is not None and factory.train_augments_append
     fake_loaders_split_config = copy.deepcopy(class_split_config["loaders"])
     del fake_loaders_split_config["train_split"]
     del fake_loaders_split_config["valid_split"]
     del fake_loaders_split_config["test_split"]
     with pytest.raises(AssertionError):
-        _ = thelper.data.loaders._LoaderFactory(fake_loaders_split_config)
+        _ = thelper.data.loaders.LoaderFactory(fake_loaders_split_config)
     normalize_query = mocker.patch("thelper.utils.query_yes_no", return_value=True)
     denormalized_loaders_config = copy.deepcopy(class_split_config["loaders"])
     denormalized_loaders_config["train_split"]["dataset_A"] = 1.5
-    factory = thelper.data.loaders._LoaderFactory(denormalized_loaders_config)
+    factory = thelper.data.loaders.LoaderFactory(denormalized_loaders_config)
     assert math.isclose(factory.train_split["dataset_A"] + factory.valid_split["dataset_A"] + factory.test_split["dataset_A"], 1)
     denormalized_loaders_config["train_split"]["dataset_A"] = 0.1
-    factory = thelper.data.loaders._LoaderFactory(denormalized_loaders_config)
+    factory = thelper.data.loaders.LoaderFactory(denormalized_loaders_config)
     assert math.isclose(factory.train_split["dataset_A"] + factory.valid_split["dataset_A"] + factory.test_split["dataset_A"], 1)
     assert normalize_query.call_count == 1
     loaders_collate_config = copy.deepcopy(class_split_config["loaders"])
     loaders_collate_config["collate_fn"] = 0
     with pytest.raises(AssertionError):
-        _ = thelper.data.loaders._LoaderFactory(loaders_collate_config)
+        _ = thelper.data.loaders.LoaderFactory(loaders_collate_config)
     loaders_collate_config["collate_fn"] = collate_fn
-    factory = thelper.data.loaders._LoaderFactory(loaders_collate_config)
+    factory = thelper.data.loaders.LoaderFactory(loaders_collate_config)
     assert factory.train_collate_fn == collate_fn
     loaders_collate_config["collate_fn"] = {
         "type": "torch.utils.data.dataloader.default_collate"
     }
-    factory = thelper.data.loaders._LoaderFactory(loaders_collate_config)
+    factory = thelper.data.loaders.LoaderFactory(loaders_collate_config)
     assert factory.train_collate_fn == torch.utils.data.dataloader.default_collate
     loaders_collate_config["collate_fn"] = "torch.utils.data.dataloader.default_collate"
-    factory = thelper.data.loaders._LoaderFactory(loaders_collate_config)
+    factory = thelper.data.loaders.LoaderFactory(loaders_collate_config)
     assert factory.train_collate_fn == torch.utils.data.dataloader.default_collate
     loaders_collate_config["train_collate_fn"] = "torch.utils.data.dataloader.default_collate"
     with pytest.raises(AssertionError):
-        _ = thelper.data.loaders._LoaderFactory(loaders_collate_config)
+        _ = thelper.data.loaders.LoaderFactory(loaders_collate_config)
     loaders_seed_config = copy.deepcopy(class_split_config["loaders"])
     loaders_seed_config["test_seed"] = 0
     loaders_seed_config["valid_seed"] = 1
     loaders_seed_config["torch_seed"] = 2
     loaders_seed_config["numpy_seed"] = 3
     loaders_seed_config["random_seed"] = "4"
     with pytest.raises(AssertionError):
-        _ = thelper.data.loaders._LoaderFactory(loaders_seed_config)
+        _ = thelper.data.loaders.LoaderFactory(loaders_seed_config)
     loaders_seed_config["random_seed"] = 4
-    factory = thelper.data.loaders._LoaderFactory(loaders_seed_config)
+    factory = thelper.data.loaders.LoaderFactory(loaders_seed_config)
     assert factory.seeds["test"] == 0
     assert factory.seeds["valid"] == 1
     assert factory.seeds["torch"] == 2