Github Actions (#235)

* remove travis

* update to pytorch 1.8

* change default frames in classic script

* update CI torch version to 1.8

* change github action names

* change CI python versions

* update action names again

.github/workflows/python-package.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: [3.6, 3.7, 3.8]
 
     steps:
     - uses: actions/checkout@v2
@@ -27,12 +27,12 @@ jobs:
       run: |
         sudo apt-get install swig
         sudo apt-get install unrar
-        pip install torch==1.5.1+cpu -f https://download.pytorch.org/whl/torch_stable.html
+        pip install torch==1.8.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
         make install
         AutoROM -v
-    - name: Linting
+    - name: Lint code
       run: |
         make lint
-    - name: Unit and integration testing
+    - name: Run tests
       run: |
         make test

all/environments/multiagent_atari.py

@@ -19,14 +19,14 @@ class MultiagentAtariEnv(MultiagentPettingZooEnv):
         device (optional): the device on which tensors will be stored
     '''
 
-    def __init__(self, env_name, device='cuda'):
-        env = self._load_env(env_name)
+    def __init__(self, env_name, device='cuda', **pettingzoo_params):
+        env = self._load_env(env_name, pettingzoo_params)
         super().__init__(env, name=env_name, device=device)
 
-    def _load_env(self, env_name):
+    def _load_env(self, env_name, pettingzoo_params):
         from pettingzoo import atari
         from supersuit import resize_v0, frame_skip_v0, reshape_v0, max_observation_v0
-        env = importlib.import_module('pettingzoo.atari.{}'.format(env_name)).env(obs_type='grayscale_image')
+        env = importlib.import_module('pettingzoo.atari.{}'.format(env_name)).env(obs_type='grayscale_image', **pettingzoo_params)
         env = max_observation_v0(env, 2)
         env = frame_skip_v0(env, 4)
         env = resize_v0(env, 84, 84)

all/experiments/multiagent_env_experiment_test.py

@@ -80,12 +80,18 @@ def test_writes_test_returns(self):
         experiment.train(episodes=3)
         experiment._writer.data = {}
         experiment.test(episodes=3)
-        self.assertEqual(experiment._writer.data, {
-            'evaluation/first_0/returns-test/mean': {'values': [856.6666666666666], 'steps': [7288]},
-            'evaluation/first_0/returns-test/std': {'values': [2.357022603955158], 'steps': [7288]},
-            'evaluation/second_0/returns-test/mean': {'values': [731.6666666666666], 'steps': [7288]},
-            'evaluation/second_0/returns-test/std': {'values': [18.856180831641264], 'steps': [7288]}
-        })
+        self.assertEqual(list(experiment._writer.data.keys()), [
+            'evaluation/first_0/returns-test/mean',
+            'evaluation/first_0/returns-test/std',
+            'evaluation/second_0/returns-test/mean',
+            'evaluation/second_0/returns-test/std'
+        ])
+        steps = experiment._writer.data['evaluation/first_0/returns-test/mean']['steps'][0]
+        for datum in experiment._writer.data.values():
+            self.assertEqual(len(datum['values']), 1)
+            self.assertGreaterEqual(datum['values'][0], 0.0)
+            self.assertEqual(len(datum['steps']), 1)
+            self.assertEqual(datum['steps'][0], steps)
 
     def test_writes_loss(self):
         experiment = MockExperiment(self.make_preset(), self.env, quiet=True, write_loss=True, save_freq=float('inf'))

all/presets/continuous/ddpg.py

@@ -63,7 +63,7 @@ def __init__(self, env, name, device, **hyperparameters):
         self.action_space = env.action_space
 
     def agent(self, writer=DummyWriter(), train_steps=float('inf')):
-        n_updates = (train_steps - self.hyperparameters["replay_start_size"]) // self.hyperparameters["update_frequency"]
+        n_updates = (train_steps - self.hyperparameters["replay_start_size"]) / self.hyperparameters["update_frequency"]
 
         q_optimizer = Adam(self.q_model.parameters(), lr=self.hyperparameters["lr_q"])
 

all/presets/continuous/sac.py

@@ -74,7 +74,7 @@ def __init__(self, env, name, device, **hyperparameters):
         self.action_space = env.action_space
 
     def agent(self, writer=DummyWriter(), train_steps=float('inf')):
-        n_updates = (train_steps - self.hyperparameters["replay_start_size"]) // self.hyperparameters["update_frequency"]
+        n_updates = (train_steps - self.hyperparameters["replay_start_size"]) / self.hyperparameters["update_frequency"]
 
         q_1_optimizer = Adam(self.q_1_model.parameters(), lr=self.hyperparameters["lr_q"])
         q_1 = QContinuous(

docs/source/guide/getting_started.rst

@@ -4,7 +4,7 @@ Getting Started
 Prerequisites
 -------------
 
-The Autonomous Learning Library requires a recent version of PyTorch (>= 10.3).
+The Autonomous Learning Library requires a recent version of PyTorch (~=1.8.0 recommended).
 Additionally, Tensorboard is required in order to enable logging.
 We also strongly recommend using a machine with a fast GPU (at minimum a GTX 970 or better, a GTX 1080ti or better is preferred).
 

integration/multiagent_atari_test.py

@@ -20,15 +20,15 @@
 
 class TestMultiagentAtariPresets(unittest.TestCase):
     def test_independent(self):
-        env = MultiagentAtariEnv('pong_v1', device=CPU)
+        env = MultiagentAtariEnv('pong_v1', max_cycles=1000, device=CPU)
         presets = {
             agent_id: dqn.device(CPU).env(env.subenvs[agent_id]).build()
             for agent_id in env.agents
         }
         validate_multiagent(IndependentMultiagentPreset('independent', CPU, presets), env)
 
     def test_independent_cuda(self):
-        env = MultiagentAtariEnv('pong_v1', device=CUDA)
+        env = MultiagentAtariEnv('pong_v1', max_cycles=1000, device=CUDA)
         presets = {
             agent_id: dqn.device(CUDA).env(env.subenvs[agent_id]).build()
             for agent_id in env.agents

scripts/classic.py

@@ -16,7 +16,7 @@ def main():
         help="The name of the device to run the agent on (e.g. cpu, cuda, cuda:0).",
     )
     parser.add_argument(
-        "--frames", type=int, default=20000, help="The number of training frames."
+        "--frames", type=int, default=50000, help="The number of training frames."
     )
     parser.add_argument(
         "--render", action="store_true", default=False, help="Render the environment."

setup.py

@@ -38,7 +38,7 @@
 
 setup(
     name="autonomous-learning-library",
-    version="0.6.2",
+    version="0.7.0-dev",
     description=("A library for building reinforcement learning agents in Pytorch"),
     packages=find_packages(),
     url="https://github.com/cpnota/autonomous-learning-library.git",
@@ -62,7 +62,7 @@
         "numpy>=1.18.0",           # math library
         "matplotlib>=3.3.0",       # plotting library
         "opencv-python~=3.4.0",      # used by atari wrappers
-        "torch~=1.5.1",            # core deep learning library
+        "torch~=1.8.0",            # core deep learning library
         "tensorboard>=2.3.0",      # logging and visualization
         "tensorboardX>=2.1.0",     # tensorboard/pytorch compatibility
         "cloudpickle>=1.2.0",      # used to copy environments