From 8a897b08bd3e491165178c6cf5b32caac2bc6d51 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 16:12:07 +0800 Subject: [PATCH 1/9] fixed unpicklable store bug --- maro/rl/storage/column_based_store.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index 8709a8d8e..8cc2010ae 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -53,6 +53,15 @@ def __next__(self): def __getitem__(self, index: int): return {k: lst[index] for k, lst in self._store.items()} + def __getstate__(self): + """A small modification to make the object picklable. + Using the default ``__dict__`` would make the object unpicklable due to the lambda function involved in the + ``defaultdict`` definition of the ``_store`` attribute. + """ + obj_dict = self.__dict__ + obj_dict["_store"] = dict(obj_dict["_store"]) + return obj_dict + @property def capacity(self): """Store capacity. From 6d27afa1ed2138cadf2061d6c03a3c2f81da2e0a Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 16:18:14 +0800 Subject: [PATCH 2/9] fixed a bug --- maro/rl/agent/abs_agent.py | 1 + 1 file changed, 1 insertion(+) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 0aa9afd76..20470f93e 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -87,5 +87,6 @@ def dump_model_dict(self, dir_path: str): def dump_experience_store(self, dir_path: str): """Dump the experience pool to disk.""" + os.makedirs(dir_path, exist_ok=True) with open(os.path.join(dir_path, self._name)) as fp: pickle.dump(self._experience_pool, fp) From bebd3939313cd03f92a6583256b4dc996fcdc2fa Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 16:19:54 +0800 Subject: [PATCH 3/9] fixed a bug --- maro/rl/agent/abs_agent.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 20470f93e..dc0ced887 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -87,6 +87,7 @@ def dump_model_dict(self, dir_path: str): def dump_experience_store(self, dir_path: str): """Dump the experience pool to disk.""" - os.makedirs(dir_path, exist_ok=True) - with open(os.path.join(dir_path, self._name)) as fp: + path = os.path.join(dir_path, self._name) + os.makedirs(path, exist_ok=True) + with open(path, "w") as fp: pickle.dump(self._experience_pool, fp) From 1a6a04be71e9db4ea9a717ee243549354cb4fd7b Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 16:27:39 +0800 Subject: [PATCH 4/9] fixed a bug --- maro/rl/agent/abs_agent.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index dc0ced887..e7e600874 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -87,7 +87,6 @@ def dump_model_dict(self, dir_path: str): def dump_experience_store(self, dir_path: str): """Dump the experience pool to disk.""" - path = os.path.join(dir_path, self._name) - os.makedirs(path, exist_ok=True) - with open(path, "w") as fp: + os.makedirs(dir_path, exist_ok=True) + with open(os.path.join(dir_path, self._name), "wb") as fp: pickle.dump(self._experience_pool, fp) From be20fd21624b69a2cf2ceb62d202f80ecc31579d Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 16:33:21 +0800 Subject: [PATCH 5/9] fixed lint formatting --- maro/rl/agent/abs_agent.py | 8 +++----- maro/rl/storage/column_based_store.py | 7 ++++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index e7e600874..5b86415b7 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -27,11 +27,9 @@ class AbsAgent(ABC): choosing actions and optimizing models. experience_pool (AbsStore): A data store that stores experiences generated by the experience shaper. """ - def __init__(self, - name: str, - algorithm: AbsAlgorithm, - experience_pool: AbsStore - ): + def __init__( + self, name: str, algorithm: AbsAlgorithm, experience_pool: AbsStore + ): self._name = name self._algorithm = algorithm self._experience_pool = experience_pool diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index 8cc2010ae..10481e42d 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -101,8 +101,9 @@ def put(self, contents: dict, overwrite_indexes: Sequence = None) -> List[int]: self._size += added_size return list(range(self._size - added_size, self._size)) else: - write_indexes = get_update_indexes(self._size, added_size, self._capacity, self._overwrite_type, - overwrite_indexes=overwrite_indexes) + write_indexes = get_update_indexes( + self._size, added_size, self._capacity, self._overwrite_type, overwrite_indexes=overwrite_indexes + ) self.update(write_indexes, contents) self._size = min(self._capacity, self._size + added_size) return write_indexes @@ -134,7 +135,7 @@ def apply_multi_filters(self, filters: Sequence[Callable]): Args: filters (Sequence[Callable]): Filter list, each item is a lambda function, - e.g., [lambda d: d['a'] == 1 and d['b'] == 1]. + e.g., [lambda d: d['a'] == 1 and d['b'] == 1]. Returns: Filtered indexes and corresponding objects. """ From 4b3daa4be3d861ef128b3606f293ac798f27afe3 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 16:38:56 +0800 Subject: [PATCH 6/9] fixed a lint formatting issue --- maro/rl/agent/abs_agent.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 5b86415b7..7ffaf82ed 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -80,8 +80,10 @@ def load_model_dict_from_file(self, file_path): def dump_model_dict(self, dir_path: str): """Dump models to disk.""" - torch.save({model_key: model.state_dict() for model_key, model in self._algorithm.model_dict.items()}, - os.path.join(dir_path, self._name)) + torch.save( + {model_key: model.state_dict() for model_key, model in self._algorithm.model_dict.items()}, + os.path.join(dir_path, self._name) + ) def dump_experience_store(self, dir_path: str): """Dump the experience pool to disk.""" From 2579543ae56f958576b605c5afd1fc4b4782f03a Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 16:40:56 +0800 Subject: [PATCH 7/9] fixed a bug --- examples/cim/dqn/multi_process_launcher.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/cim/dqn/multi_process_launcher.py b/examples/cim/dqn/multi_process_launcher.py index 9ec989550..c23075618 100644 --- a/examples/cim/dqn/multi_process_launcher.py +++ b/examples/cim/dqn/multi_process_launcher.py @@ -11,7 +11,7 @@ ACTOR_NUM = config.distributed.learner.peer["actor_worker"] # must be same as in config -LEARNER_NUM = config.distributed.actor.peer["actor"] +LEARNER_NUM = config.distributed.actor.peer["learner"] learner_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_learner.py &" actor_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_actor.py &" From 10c7feaf9c5caf8f048dec64e81ea54a5ccb8449 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Thu, 22 Oct 2020 16:58:22 +0800 Subject: [PATCH 8/9] renamed dump_experience_store to dump_experience_pool --- examples/cim/dqn/multi_process_launcher.py | 2 +- maro/rl/agent/abs_agent.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/cim/dqn/multi_process_launcher.py b/examples/cim/dqn/multi_process_launcher.py index c23075618..9ec989550 100644 --- a/examples/cim/dqn/multi_process_launcher.py +++ b/examples/cim/dqn/multi_process_launcher.py @@ -11,7 +11,7 @@ ACTOR_NUM = config.distributed.learner.peer["actor_worker"] # must be same as in config -LEARNER_NUM = config.distributed.actor.peer["learner"] +LEARNER_NUM = config.distributed.actor.peer["actor"] learner_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_learner.py &" actor_path = f"{os.path.split(os.path.realpath(__file__))[0]}/dist_actor.py &" diff --git a/maro/rl/agent/abs_agent.py b/maro/rl/agent/abs_agent.py index 7ffaf82ed..03967875d 100644 --- a/maro/rl/agent/abs_agent.py +++ b/maro/rl/agent/abs_agent.py @@ -85,7 +85,7 @@ def dump_model_dict(self, dir_path: str): os.path.join(dir_path, self._name) ) - def dump_experience_store(self, dir_path: str): + def dump_experience_pool(self, dir_path: str): """Dump the experience pool to disk.""" os.makedirs(dir_path, exist_ok=True) with open(os.path.join(dir_path, self._name), "wb") as fp: From ac52e89fb2e4fd7cedf53bf02525e6fb4cb220a2 Mon Sep 17 00:00:00 2001 From: ysqyang Date: Tue, 27 Oct 2020 11:02:13 +0800 Subject: [PATCH 9/9] fixed a PR comment --- maro/rl/storage/column_based_store.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/maro/rl/storage/column_based_store.py b/maro/rl/storage/column_based_store.py index 10481e42d..46398185e 100644 --- a/maro/rl/storage/column_based_store.py +++ b/maro/rl/storage/column_based_store.py @@ -54,7 +54,7 @@ def __getitem__(self, index: int): return {k: lst[index] for k, lst in self._store.items()} def __getstate__(self): - """A small modification to make the object picklable. + """A patch for picking the object with lambda. Using the default ``__dict__`` would make the object unpicklable due to the lambda function involved in the ``defaultdict`` definition of the ``_store`` attribute. """