real object nav (supercedes objnav refactor)

dev_requirements.txt

@@ -0,0 +1,186 @@
+absl-py==0.12.0
+addict==2.4.0
+# ai2thor==0+dc0f9ecd8672dc2d62651f567ff95c63f3542332
+aiohttp==3.8.1
+aiosignal==1.2.0
+allenact==0.4.0
+allenact-plugins==0.4.0
+antlr4-python3-runtime==4.8
+appdirs==1.4.4
+appnope==0.1.2
+argon2-cffi==20.1.0
+astunparse==1.6.3
+async-generator==1.10
+async-timeout==4.0.2
+asynctest==0.13.0
+attrs==20.3.0
+aws-requests-auth==0.4.3
+backcall==0.2.0
+black==21.4b2
+bleach==3.3.0
+botocore==1.20.67
+cachetools==4.2.2
+certifi==2020.12.5
+cffi==1.14.5
+chardet==4.0.0
+charset-normalizer==2.0.12
+click==7.1.2
+clip @ git+https://github.com/openai/CLIP.git@04f4dc2ca1ed0acc9893bd1a3b526a7e02c4bb10
+cloudpickle==1.6.0
+cmake==3.18.4.post1
+colour==0.1.5
+cycler==0.10.0
+Cython==0.29.23
+dataclasses>0.5
+datasets==1.18.4
+decorator==4.4.2
+defusedxml==0.7.1
+# detectron2 @ git+https://github.com/facebookresearch/detectron2.git@d285dea0a58675217b061f48182dc031d2e3b677
+dill==0.3.4
+entrypoints==0.3
+filelock==3.0.12
+Flask==1.1.2
+flatbuffers==1.12
+frozenlist==1.2.0
+fsspec==2022.1.0
+ftfy==6.0.3
+future==0.18.2
+fvcore==0.1.5.post20210609
+gast==0.3.3
+gin==0.1.6
+gin-config==0.4.0
+google-auth==1.30.0
+google-auth-oauthlib==0.4.4
+google-pasta==0.2.0
+grpcio==1.32.0
+gym==0.17.3
+h5py==2.10.0
+huggingface-hub==0.4.0
+hydra-core==1.1.0
+idna==2.10
+idna-ssl==1.1.0
+imageio==2.9.0
+imageio-ffmpeg==0.4.3
+importlib-metadata==4.0.1
+importlib-resources==5.1.4
+iopath==0.1.8
+ipykernel==5.5.4
+ipython==7.16.1
+ipython-genutils==0.2.0
+ipywidgets==7.6.3
+itsdangerous==1.1.0
+jedi==0.17.0
+Jinja2==2.11.3
+jmespath==0.10.0
+joblib==1.1.0
+jsonschema==3.2.0
+jupyter==1.0.0
+jupyter-client==6.1.12
+jupyter-console==6.4.0
+jupyter-core==4.7.1
+jupyterlab-pygments==0.1.2
+jupyterlab-widgets==1.0.0
+Keras-Preprocessing==1.1.2
+kiwisolver==1.3.1
+llvmlite==0.36.0
+Markdown==3.3.4
+MarkupSafe==1.1.1
+matplotlib==3.3.4
+mistune==0.8.4
+moviepy==1.0.3
+msgpack==1.0.2
+multidict==5.2.0
+multiprocess==0.70.12.2
+mypy-extensions==0.4.3
+nbclient==0.5.3
+nbconvert==6.0.7
+nbformat==5.1.3
+nest-asyncio==1.5.1
+networkx==2.5.1
+notebook==6.3.0
+numba==0.53.1
+numpy==1.19.5
+numpy-quaternion==2021.4.5.14.42.35
+oauthlib==3.1.0
+omegaconf==2.1.0
+open3d==0.13.0
+opencv-contrib-python==4.5.5.64
+opt-einsum==3.3.0
+packaging==20.9
+pandas==1.1.5
+pandocfilters==1.4.3
+parso==0.8.2
+pathspec==0.8.1
+pexpect==4.8.0
+pickleshare==0.7.5
+Pillow==8.2.0
+portalocker==2.3.0
+proglog==0.1.9
+progressbar2==3.53.1
+prometheus-client==0.10.1
+prompt-toolkit==3.0.18
+protobuf==3.15.8
+ptyprocess==0.7.0
+pyarrow==6.0.1
+pyasn1==0.4.8
+pyasn1-modules==0.2.8
+pycocotools==2.0.2
+pycparser==2.20
+pydot==1.4.2
+pyglet==1.5.0
+Pygments==2.9.0
+pyparsing==2.4.7
+pyquaternion==0.9.9
+pyrsistent==0.17.3
+python-dateutil==2.8.1
+python-utils==2.5.6
+python-xlib==0.29
+pytz==2021.3
+PyWavelets==1.1.1
+PyYAML==5.4.1
+pyzmq==22.0.3
+qtconsole==5.1.0
+QtPy==1.9.0
+regex==2021.4.4
+requests==2.25.1
+requests-oauthlib==1.3.0
+responses==0.17.0
+rsa==4.7.2
+scikit-image==0.17.2
+scikit-learn==0.24.2
+scipy==1.5.4
+Send2Trash==1.5.0
+setproctitle==1.2.2
+Shapely==1.8.1.post1
+six==1.15.0
+tabulate==0.8.9
+tensorboard==2.5.0
+tensorboard-data-server==0.6.1
+tensorboard-plugin-wit==1.8.0
+tensorboardX==2.2
+tensorflow==2.4.1
+tensorflow-estimator==2.4.0
+termcolor==1.1.0
+terminado==0.9.4
+testpath==0.4.4
+threadpoolctl==3.0.0
+tifffile==2020.9.3
+toml==0.10.2
+torch==1.8.1
+torchaudio==0.8.1
+torchvision==0.9.1
+tornado==6.1
+tqdm==4.63.0
+traitlets==4.3.3
+typed-ast==1.4.3
+typing-extensions==3.7.4.3
+urllib3==1.26.4
+wcwidth==0.2.5
+webencodings==0.5.1
+Werkzeug==1.0.1
+widgetsnbextension==3.5.1
+wrapt==1.12.1
+xxhash==3.0.0
+yacs==0.1.8
+yarl==1.7.2
+zipp==3.4.1

ithor_arm/ithor_arm_environment.py

@@ -1,17 +1,22 @@
 """A wrapper for engaging with the ManipulaTHOR environment."""
 
+from ast import For
 import copy
 import math
 import typing
 import warnings
 from typing import Tuple, Dict, List, Set, Union, Any, Optional
 
 import ai2thor.server
+from cv2 import USAGE_DEFAULT
 import numpy as np
 from ai2thor.controller import Controller
 from allenact_plugins.ithor_plugin.ithor_constants import VISIBILITY_DISTANCE, FOV
 from allenact_plugins.ithor_plugin.ithor_environment import IThorEnvironment
 
+from ithor_arm.ithor_arm_noise_models import NoiseInMotionHabitatFlavor, NoiseInMotionSimple1DNormal
+from ithor_arm.arm_calculation_utils import convert_world_to_agent_coordinate
+
 from ithor_arm.ithor_arm_constants import (
     ADITIONAL_ARM_ARGS,
     ARM_MIN_HEIGHT,
@@ -56,6 +61,7 @@ def __init__(
             verbose: bool = False,
             env_args=None,
     ) -> None:
+
         """Initializer.
 
         # Parameters
@@ -84,6 +90,9 @@ def __init__(
         simplify_physics : Whether or not to simplify physics when applicable. Currently this only simplies object
             interactions when opening drawers (when simplified, objects within a drawer do not slide around on
             their own when the drawer is opened or closed, instead they are effectively glued down).
+        *_noise_meta_dist_params : [mean, variance] defines the normal distribution over which the actual noise parameters
+            for a motion noise distribution will be drawn. Distributions for noise in motion will be re-rolled every scene 
+            reset with new bias and variance values drawn from these meta-distributions.
         """
 
         self._start_player_screen_width = player_screen_width
@@ -112,6 +121,21 @@ def __init__(
         self._always_return_visible_range = False
         self.simplify_physics = simplify_physics
 
+        if 'motion_noise_type' in env_args.keys():
+            if env_args['motion_noise_type'] == 'habitat':
+                self.noise_model = NoiseInMotionHabitatFlavor(**env_args['motion_noise_args'])
+            elif env_args['motion_noise_type'] == 'simple1d':
+                self.noise_model = NoiseInMotionSimple1DNormal(**env_args['motion_noise_args'])
+            else:
+                print('Unrecognized motion noise model type')
+                ForkedPdb().set_trace()
+        else:
+            self.noise_model = NoiseInMotionHabitatFlavor(effect_scale=0.0) # un-noise model
+
+        self.ahead_nominal = 0.2
+        self.rotate_nominal = 45
+
+
         self.start(None)
         # self.check_controller_version()
 
@@ -124,6 +148,16 @@ def __init__(
             self.env_args["quality"] = self._quality
         # self.memory_frames = []
 
+    def check_controller_version(self):
+        if MANIPULATHOR_COMMIT_ID is not None:
+            assert (
+                    MANIPULATHOR_COMMIT_ID in self.controller._build.url
+            ), "Build number is not right, {} vs {}, use  pip3 install -e git+https://github.com/allenai/ai2thor.git@{}#egg=ai2thor".format(
+                self.controller._build.url,
+                MANIPULATHOR_COMMIT_ID,
+                MANIPULATHOR_COMMIT_ID,
+            )
+
     # def check_controller_version(self):
     #     if MANIPULATHOR_COMMIT_ID is not None:
     #         assert (
@@ -222,6 +256,10 @@ def reset(
 
         self.list_of_actions_so_far = []
 
+        self.noise_model.reset_noise_model()
+        self.nominal_agent_location = self.get_agent_location()
+
+
     def randomize_agent_location(
             self, seed: int = None, partial_position: Optional[Dict[str, float]] = None
     ) -> Dict:
@@ -299,6 +337,36 @@ def get_absolute_hand_state(self):
         xyz_dict = self.correct_nan_inf(xyz_dict, "absolute hand")
         return dict(position=xyz_dict, rotation={"x": 0, "y": 0, "z": 0})
 
+
+    def update_nominal_location(self, action_dict):
+        # location = {
+        #     "x": metadata["agent"]["position"]["x"],
+        #     "y": metadata["agent"]["position"]["y"],
+        #     "z": metadata["agent"]["position"]["z"],
+        #     "rotation": metadata["agent"]["rotation"]["y"],
+        #     "horizon": metadata["agent"]["cameraHorizon"],
+        #     "standing": metadata.get("isStanding", metadata["agent"].get("isStanding")),
+        # }
+
+        curr_loc = self.nominal_agent_location
+        new_loc = copy.deepcopy(curr_loc)
+
+        if action_dict['action'] is 'RotateLeft':
+            new_loc["rotation"] = (new_loc["rotation"] - self.rotate_nominal) % 360
+        elif action_dict['action'] is 'RotateRight':
+            new_loc["rotation"] = (new_loc["rotation"] + self.rotate_nominal) % 360
+        elif action_dict['action'] is 'MoveAhead':
+            new_loc["x"] += self.ahead_nominal * np.sin(new_loc["rotation"] * np.pi / 180)
+            new_loc["z"] += self.ahead_nominal * np.cos(new_loc["rotation"] * np.pi / 180)
+        elif action_dict['action'] is 'TeleportFull':
+            new_loc["x"] = action_dict['x']
+            new_loc["y"] = action_dict['y']
+            new_loc["z"] = action_dict['z']
+            new_loc["rotation"] = action_dict['rotation']['y']
+            new_loc["horizon"] = action_dict['horizon']
+
+        self.nominal_agent_location = new_loc
+
     def get_pickupable_objects(self):
 
         event = self.controller.last_event
@@ -361,6 +429,7 @@ def step(
     ) -> ai2thor.server.Event:
         """Take a step in the ai2thor environment."""
         action = typing.cast(str, action_dict["action"])
+        original_action_dict = copy.deepcopy(action_dict)
 
         skip_render = "renderImage" in action_dict and not action_dict["renderImage"]
         last_frame: Optional[np.ndarray] = None
@@ -396,18 +465,41 @@ def step(
 
             copy_aditions = copy.deepcopy(ADITIONAL_ARM_ARGS)
 
+            # RH: order matters, nominal action happens last
             action_dict = {**action_dict, **copy_aditions}
             if action in [MOVE_AHEAD]:
+                noise = self.noise_model.get_ahead_drift(self.ahead_nominal)
+
+                action_dict["action"] = "RotateAgent"
+                action_dict["degrees"] = noise[2]
+                sr = self.controller.step(action_dict)
+
+                action_dict = dict()
                 action_dict["action"] = "MoveAgent"
-                action_dict["ahead"] = 0.2
+                action_dict["ahead"] = self.ahead_nominal + noise[0]
+                action_dict["right"] = noise[1]
 
             elif action in [ROTATE_RIGHT]:
+                noise = self.noise_model.get_rotate_drift()
+                action_dict["action"] = "MoveAgent"
+                action_dict["ahead"] = noise[0]
+                action_dict["right"] = noise[1]
+                sr = self.controller.step(action_dict)
+
+                action_dict = dict()
                 action_dict["action"] = "RotateAgent"
-                action_dict["degrees"] = 45
+                action_dict["degrees"] = noise[2] + self.rotate_nominal
 
             elif action in [ROTATE_LEFT]:
+                noise = self.noise_model.get_rotate_drift()
+                action_dict["action"] = "MoveAgent"
+                action_dict["ahead"] = noise[0]
+                action_dict["right"] = noise[1]
+                sr = self.controller.step(action_dict)
+
+                action_dict = dict()
                 action_dict["action"] = "RotateAgent"
-                action_dict["degrees"] = -45
+                action_dict["degrees"] = noise[2] - self.rotate_nominal
 
         elif "MoveArm" in action:
             copy_aditions = copy.deepcopy(ADITIONAL_ARM_ARGS)
@@ -441,10 +533,13 @@ def step(
                     k: v for (k, v) in base_position.items() if k in ["x", "y", "z"]
                 }
 
-
-
         sr = self.controller.step(action_dict)
         self.list_of_actions_so_far.append(action_dict)
+
+        # RH: Nominal location only updates for successful actions. Note that that drift 
+        # action might succeed even if the "main" action fails
+        if sr.metadata["lastActionSuccess"]:
+            self.update_nominal_location(original_action_dict)
 
         # if action in SET_OF_ALL_AGENT_ACTIONS:
         #     self.update_memory()