-
Notifications
You must be signed in to change notification settings - Fork 50
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Adding default agentmode for ithor objectnav task #307
base: main
Are you sure you want to change the base?
Changes from 13 commits
53b8ca6
dfb5641
877f7a5
0d33b2b
66a5b03
668b3ca
7d8a136
aadc4a1
24fb1d1
37795be
6a72649
90ef4f5
a7c7f0d
6a07db0
fe3ed2e
be37e2a
a76d94f
11bc7cd
5f86224
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -1,6 +1,8 @@ | ||
import copy | ||
import random | ||
from typing import List, Dict, Optional, Any, Union, cast | ||
import gzip | ||
import json | ||
from typing import List, Optional, Union, Dict, Any, cast, Tuple | ||
|
||
import gym | ||
|
||
|
@@ -198,3 +200,218 @@ def set_seed(self, seed: int): | |
self.seed = seed | ||
if seed is not None: | ||
set_seed(seed) | ||
|
||
|
||
class ObjectNavDatasetTaskSampler(TaskSampler): | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can we call this There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. It seems like a large amount of functionality is shared between this task sampler and the robothor variant. Can we maybe create an abstract superclass for both of these task samplers (containing the shared functions) and have them both subclass it? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I like the idea. We could have that under an There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I have renamed it to For the abstract superclass suggestion, should I create a file There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. @kshitijd20 - so that you can merge this and work on other things, let's just keep it as is for now. It would be great if you could add a comment like: # TODO: Merge functionality of `ObjectNavDatasetTaskSampler` for iTHOR and RoboTHOR. somewhere there. @jordis-ai2 - I hadn't thought of going as far as moving it to There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. That sounds really great. As soon as I'm done with my current tasks, I think I will give this priority. If things are done correctly, we might remove quite a bit of lines of code 👍 |
||
def __init__( | ||
self, | ||
scenes: List[str], | ||
scene_directory: str, | ||
sensors: List[Sensor], | ||
max_steps: int, | ||
env_args: Dict[str, Any], | ||
action_space: gym.Space, | ||
# rewards_config: Dict, | ||
seed: Optional[int] = None, | ||
deterministic_cudnn: bool = False, | ||
loop_dataset: bool = True, | ||
allow_flipping=False, | ||
env_class=IThorEnvironment, | ||
**kwargs, | ||
) -> None: | ||
# self.rewards_config = rewards_config | ||
self.env_args = env_args | ||
self.scenes = scenes | ||
self.episodes = { | ||
scene: ObjectNavDatasetTaskSampler.load_dataset(scene, scene_directory) | ||
for scene in scenes | ||
} | ||
self.env_class = env_class | ||
self.object_types = [ | ||
ep["object_type"] for scene in self.episodes for ep in self.episodes[scene] | ||
] | ||
self.env: Optional[IThorEnvironment] = None | ||
self.sensors = sensors | ||
self.max_steps = max_steps | ||
self._action_space = action_space | ||
self.allow_flipping = allow_flipping | ||
self.scene_counter: Optional[int] = None | ||
self.scene_order: Optional[List[str]] = None | ||
self.scene_id: Optional[int] = None | ||
# get the total number of tasks assigned to this process | ||
if loop_dataset: | ||
self.max_tasks = None | ||
else: | ||
self.max_tasks = sum(len(self.episodes[scene]) for scene in self.episodes) | ||
self.reset_tasks = self.max_tasks | ||
self.scene_index = 0 | ||
self.episode_index = 0 | ||
|
||
self._last_sampled_task: Optional[ObjectNaviThorGridTask] = None | ||
|
||
self.seed: Optional[int] = None | ||
self.set_seed(seed) | ||
|
||
if deterministic_cudnn: | ||
set_deterministic_cudnn() | ||
|
||
self.reset() | ||
|
||
def _create_environment(self) -> IThorEnvironment: | ||
env = self.env_class( | ||
make_agents_visible=False, | ||
object_open_speed=0.05, | ||
restrict_to_initially_reachable_points=False, | ||
**self.env_args, | ||
) | ||
return env | ||
|
||
@staticmethod | ||
def load_dataset(scene: str, base_directory: str) -> List[Dict]: | ||
filename = ( | ||
"/".join([base_directory, scene]) | ||
if base_directory[-1] != "/" | ||
else "".join([base_directory, scene]) | ||
) | ||
filename += ".json.gz" | ||
fin = gzip.GzipFile(filename, "r") | ||
json_bytes = fin.read() | ||
fin.close() | ||
json_str = json_bytes.decode("utf-8") | ||
data = json.loads(json_str) | ||
random.shuffle(data) | ||
return data | ||
|
||
@staticmethod | ||
def load_distance_cache_from_file(scene: str, base_directory: str) -> Dict: | ||
filename = ( | ||
"/".join([base_directory, scene]) | ||
if base_directory[-1] != "/" | ||
else "".join([base_directory, scene]) | ||
) | ||
filename += ".json.gz" | ||
fin = gzip.GzipFile(filename, "r") | ||
json_bytes = fin.read() | ||
fin.close() | ||
json_str = json_bytes.decode("utf-8") | ||
data = json.loads(json_str) | ||
return data | ||
|
||
@property | ||
def __len__(self) -> Union[int, float]: | ||
"""Length. | ||
|
||
# Returns | ||
|
||
Number of total tasks remaining that can be sampled. Can be float('inf'). | ||
""" | ||
return float("inf") if self.max_tasks is None else self.max_tasks | ||
|
||
@property | ||
def total_unique(self) -> Optional[Union[int, float]]: | ||
return self.reset_tasks | ||
|
||
@property | ||
def last_sampled_task(self) -> Optional[ObjectNaviThorGridTask]: | ||
return self._last_sampled_task | ||
|
||
def close(self) -> None: | ||
if self.env is not None: | ||
self.env.stop() | ||
|
||
@property | ||
def all_observation_spaces_equal(self) -> bool: | ||
"""Check if observation spaces equal. | ||
|
||
# Returns | ||
|
||
True if all Tasks that can be sampled by this sampler have the | ||
same observation space. Otherwise False. | ||
""" | ||
return True | ||
|
||
@property | ||
def length(self) -> Union[int, float]: | ||
"""Length. | ||
|
||
# Returns | ||
|
||
Number of total tasks remaining that can be sampled. Can be float('inf'). | ||
""" | ||
return float("inf") if self.max_tasks is None else self.max_tasks | ||
|
||
def next_task( | ||
self, force_advance_scene: bool = False | ||
) -> Optional[ObjectNaviThorGridTask]: | ||
if self.max_tasks is not None and self.max_tasks <= 0: | ||
return None | ||
|
||
if self.episode_index >= len(self.episodes[self.scenes[self.scene_index]]): | ||
self.scene_index = (self.scene_index + 1) % len(self.scenes) | ||
# shuffle the new list of episodes to train on | ||
random.shuffle(self.episodes[self.scenes[self.scene_index]]) | ||
self.episode_index = 0 | ||
scene = self.scenes[self.scene_index] | ||
episode = self.episodes[scene][self.episode_index] | ||
if self.env is None: | ||
self.env = self._create_environment() | ||
|
||
if scene.replace("_physics", "") != self.env.scene_name.replace("_physics", ""): | ||
self.env.reset(scene_name=scene) | ||
else: | ||
self.env.reset_object_filter() | ||
|
||
self.env.set_object_filter( | ||
object_ids=[ | ||
o["objectId"] | ||
for o in self.env.last_event.metadata["objects"] | ||
if o["objectType"] == episode["object_type"] | ||
] | ||
) | ||
|
||
task_info = {"scene": scene, "object_type": episode["object_type"]} | ||
if len(task_info) == 0: | ||
get_logger().warning( | ||
"Scene {} does not contain any" | ||
" objects of any of the types {}.".format(scene, self.object_types) | ||
) | ||
task_info["initial_position"] = episode["initial_position"] | ||
task_info["initial_orientation"] = episode["initial_orientation"] | ||
task_info["initial_horizon"] = episode.get("initial_horizon", 0) | ||
task_info["distance_to_target"] = episode.get("shortest_path_length") | ||
task_info["path_to_target"] = episode.get("shortest_path") | ||
task_info["object_type"] = episode["object_type"] | ||
task_info["id"] = episode["id"] | ||
if self.allow_flipping and random.random() > 0.5: | ||
task_info["mirrored"] = True | ||
else: | ||
task_info["mirrored"] = False | ||
|
||
self.episode_index += 1 | ||
if self.max_tasks is not None: | ||
self.max_tasks -= 1 | ||
if not self.env.teleport( | ||
pose=episode["initial_position"], | ||
rotation=episode["initial_orientation"], | ||
horizon=episode.get("initial_horizon", 0), | ||
): | ||
return self.next_task() | ||
self._last_sampled_task = ObjectNaviThorGridTask( | ||
env=self.env, | ||
sensors=self.sensors, | ||
task_info=task_info, | ||
max_steps=self.max_steps, | ||
action_space=self._action_space, | ||
) | ||
|
||
return self._last_sampled_task | ||
|
||
def reset(self): | ||
self.episode_index = 0 | ||
self.scene_index = 0 | ||
self.max_tasks = self.reset_tasks | ||
|
||
def set_seed(self, seed: int): | ||
self.seed = seed | ||
if seed is not None: | ||
set_seed(seed) |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Why not use the existing
teleport_agent_to
function?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I tried it at first but it was giving me a lot of warnings of this type
"Teleportation FAILED but agent still moved (position_dist {}, rot diff {})"
So, I was not sure and copied the teleport function from Robothor environment.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Gotcha, it would be nice if you could track down what the issue is, otherwise I would need to be convinced that it's worth having two functions that do roughly the same thing.