Merge pull request #1 from facebookresearch/master

Pull from fb master

.circleci/config.yml

@@ -0,0 +1,148 @@
+version: 2
+gpu: &gpu
+  machine:
+    image: ubuntu-1604:201903-01
+  resource_class: gpu.small
+  environment:
+    FPS_THRESHOLD: 900
+
+
+jobs:
+  python_lint:
+    docker:
+      - image: circleci/python:3.6
+    steps:
+      - checkout
+      - run:
+          name: setup
+          command: |
+              sudo pip install -U black "isort[pyproject]" numpy pytest sphinx
+              sudo pip install -r requirements.txt
+      - run:
+          name: run black
+          command: |
+              black --version
+              black habitat_sim/. examples/. tests/. setup.py --diff
+              black habitat_sim/. examples/. tests/. setup.py --check
+      - run:
+          name: run isort
+          command: |
+              isort --version
+              isort -rc habitat_sim/. examples/. tests/. setup.py --diff
+              isort -rc habitat_sim/. examples/. tests/. setup.py --check-only
+  cpp_lint:
+    docker:
+      - image: circleci/buildpack-deps:disco
+    steps:
+      - checkout
+      - run:
+          name: setup
+          command: |
+              sudo apt-get update -y
+              sudo apt-get install -y clang-format-8
+              sudo ln -s /usr/bin/clang-format-8 /usr/bin/clang-format
+              clang-format --version
+      - run:
+          name: run clang-format
+          command: |
+            find . -not -path "*/\.*" -not -path "*/deps/*" -not -path "*/obsolete/*" -not -path "*/build/*" | grep -E ".*\.cpp$|.*\.h$|.*\.cu$|.*\.hpp$" | xargs -I {} bash -c "diff -u <(cat {}) <(clang-format -style=file {})"
+
+  build_install_test_ubuntu:
+    <<: *gpu
+    steps:
+      - checkout:
+          path: ./habitat-sim
+      - run:
+          name: Install cmake
+          no_output_timeout: 1h
+          command: |
+              wget https://github.com/Kitware/CMake/releases/download/v3.13.4/cmake-3.13.4-Linux-x86_64.sh
+              sudo mkdir /opt/cmake
+              sudo sh ./cmake-3.13.4-Linux-x86_64.sh --prefix=/opt/cmake --skip-license
+              sudo ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake
+      - run:
+          name: Install conda and dependencies
+          no_output_timeout: 1h
+          command: |
+              curl -o ~/miniconda.sh -O  https://repo.continuum.io/miniconda/Miniconda3-latest-Linux-x86_64.sh
+              chmod +x ~/miniconda.sh
+              ~/miniconda.sh -b -p $HOME/miniconda
+              rm ~/miniconda.sh
+              $HOME/miniconda/bin/conda install -y numpy pyyaml scipy ipython mkl mkl-include
+              export PATH=$HOME/miniconda/bin:$PATH
+              conda create -y -n habitat python=3.6
+              . activate habitat
+              conda install -y -c conda-forge ninja numpy pyyaml scipy ipython mkl mkl-include pytest
+              sudo apt-get update || true
+              sudo apt-get install -y --no-install-recommends \
+                  build-essential \
+                  git \
+                  curl \
+                  vim \
+                  ca-certificates \
+                  libjpeg-dev \
+                  libglm-dev \
+                  libegl1-mesa-dev \
+                  xorg-dev \
+                  freeglut3-dev \
+                  pkg-config \
+                  wget \
+                  zip \
+                  unzip || true
+      - run:
+          name: Install cuda
+          no_output_timeout: 1h
+          command: |
+              wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/cuda-repo-ubuntu1604_8.0.44-1_amd64.deb
+              sudo dpkg -i cuda-repo-ubuntu1604_8.0.44-1_amd64.deb
+              sudo apt-get update || true
+              sudo apt-get --yes --force-yes install cuda
+              nvidia-smi
+      - run:
+          name: Download test data
+          command: |
+              cd habitat-sim
+              wget http://dl.fbaipublicfiles.com/habitat/habitat-test-scenes.zip
+              unzip habitat-test-scenes.zip
+              rm habitat-test-scenes.zip
+      - run:
+          name: Build, install habitat-sim and run benchmark
+          no_output_timeout: 1h
+          command: |
+              export PATH=$HOME/miniconda/bin:$PATH
+              . activate habitat; cd habitat-sim
+              pip install -r requirements.txt
+              python setup.py install --headless
+              python examples/example.py --scene data/scene_datasets/habitat-test-scenes/van-gogh-room.glb --silent --test_fps_regression $FPS_THRESHOLD
+      - run:
+          name: Run sim tests
+          command: |
+              cd habitat-sim
+              export PATH=$HOME/miniconda/bin:$PATH
+              . activate habitat
+              pip install nbval
+              export PYTHONPATH=$PYTHONPATH:$(pwd)
+              ./build.sh --headless
+              HEADLESS=True pip install -e .
+              pytest --ignore notebooks/habitat-sim-demo.ipynb
+      - run:
+          name: Run api tests
+          command: |
+              git clone https://github.com/facebookresearch/habitat-api.git
+              export PYTHONPATH=$PYTHONPATH:$(pwd)/habitat-sim
+              export PATH=$HOME/miniconda/bin:$PATH
+              . activate habitat; cd habitat-api
+              ln -s ../habitat-sim/data data
+              conda install -c  conda-forge opencv -y
+              conda install -y pytorch torchvision cudatoolkit=10.0 -c pytorch
+              pip install -r requirements.txt
+              python setup.py test
+
+
+workflows:
+  version: 2
+  build_and_install:
+    jobs:
+      - python_lint
+      - cpp_lint
+      - build_install_test_ubuntu

.gitignore

@@ -165,7 +165,7 @@ run_example
 
 tests/17DRP5sb8fy
 data/
-habitat-test-scenes.zip
+*.zip
 habitat_sim/_ext
 
 .setuppy_args_cache.json

.pre-commit-config.yaml

@@ -2,21 +2,21 @@ exclude: 'build|src/deps|src/obsolete'
 
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v1.2.3
+    rev: v2.2.3
     hooks:
     -   id: trailing-whitespace
     -   id: check-added-large-files
         args: ['--maxkb=2000']
     -   id: end-of-file-fixer
 
 -   repo: https://github.com/pre-commit/mirrors-isort
-    rev: master
+    rev: v4.3.20
     hooks:
     -   id: isort
         language_version: python3.6
 
 -   repo: https://github.com/ambv/black
-    rev: stable
+    rev: 19.3b0
     hooks:
     - id: black
       language_version: python3.6
@@ -25,7 +25,7 @@ repos:
     hooks:
     - id: clang-format
       name: Run clang-format
-      entry: clang-format -i
+      entry: clang-format -i -style=file
       types: [text]
       files: '.*\.cpp|.*\.h'
       language: system

README.md

@@ -1,14 +1,10 @@
-<p align="center">
-  <img width = "50%" src='docs/logos/habitat_logo_with_text_horizontal_blue.png' />
-</p>
-
---------------------------------------------------------------------------------
-
-
+[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://github.com/facebookresearch/habitat-sim/blob/master/LICENSE)
+[![CircleCI](https://circleci.com/gh/facebookresearch/habitat-sim.svg?style=shield)](https://circleci.com/gh/facebookresearch/habitat-sim)
+[![PRs Welcome](https://img.shields.io/badge/PRs-welcome-brightgreen.svg)](https://github.com/facebookresearch/habitat-sim/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22)
 
 # Habitat-Sim
 
-A flexible, high-performance 3D simulator with configurable agents, multiple sensors, and generic 3D dataset handling (with built-in support for [SUNCG](http://suncg.cs.princeton.edu/), [MatterPort3D](https://niessner.github.io/Matterport/), [Gibson](http://gibsonenv.stanford.edu/database/) and other datasets).
+A flexible, high-performance 3D simulator with configurable agents, multiple sensors, and generic 3D dataset handling (with built-in support for [MatterPort3D](https://niessner.github.io/Matterport/), [Gibson](http://gibsonenv.stanford.edu/database/), [Replica](https://github.com/facebookresearch/Replica-Dataset), and other datasets).
 When rendering a scene from the Matterport3D dataset, Habitat-Sim achieves several thousand frames per second (FPS) running single-threaded, and reaches <a href="#fps_table"><b>over 10,000 FPS multi-process</b></a> on a single GPU!
 
 <p align="center">
@@ -45,8 +41,8 @@ This empowers a paradigm shift from 'internet AI' based on static datasets (e.g.
 If you use the Habitat platform in your research, please cite the following [technical report](https://arxiv.org/abs/1904.01201):
 ```
 @article{habitat19arxiv,
-  title =   {Habitat: A Platform for Embodied AI Research},
-  author =  {{Manolis Savva*}, {Abhishek Kadian*}, {Oleksandr Maksymets*}, Yili Zhao, Erik Wijmans, Bhavana Jain, Julian Straub, Jia Liu, Vladlen Koltun, Jitendra Malik, Devi Parikh and Dhruv Batra},
+  title =   {Habitat: {A} {P}latform for {E}mbodied {AI} {R}esearch},
+  author =  {{Manolis Savva*} and {Abhishek Kadian*} and {Oleksandr Maksymets*} and Yili Zhao and Erik Wijmans and Bhavana Jain and Julian Straub and Jia Liu and Vladlen Koltun and Jitendra Malik and Devi Parikh and Dhruv Batra},
   journal = {arXiv preprint arXiv:1904.01201},
   year =    {2019}
 }
@@ -188,14 +184,11 @@ We also provide a docker setup for habitat-stack, refer to [habitat-docker-setup
 
 - The full Matterport3D (MP3D) dataset for use with Habitat can be downloaded using the official [Matterport3D](https://niessner.github.io/Matterport/) download script as follows: `python download_mp.py --task habitat -o path/to/download/`. You only need the habitat zip archive and not the entire Matterport3D dataset. Note that this download script requires python 2.7 to run.
 - The Gibson dataset for use with Habitat can be downloaded by agreeing to the terms of use in the [Gibson](https://github.com/StanfordVL/GibsonEnv#database) repository
-- To obtain access to a test version of the full SUNCG dataset, please contact us.
 
 ## Examples
 
 Load a specific MP3D or Gibson house: `examples/example.py --scene path/to/mp3d/house_id.glb`.
 
-Load a specific SUNCG house: `examples/example.py --scene path/to/suncg/house_id/house.json`.
-
 Additional arguments to `example.py` are provided to change the sensor configuration, print statistics of the semantic annotations in a scene, compute action-space shortest path trajectories, and set other useful functionality. Refer to the `example.py` and `demo_runner.py` source files for an overview.
 
 To reproduce the benchmark table from above run `examples/benchmark.py --scene /path/to/mp3d/17DRP5sb8fy/17DRP5sb8fy.glb`.
@@ -208,15 +201,17 @@ To reproduce the benchmark table from above run `examples/benchmark.py --scene /
 
 ### Code style
 
-We use `clang-format` for linting and code style enforcement of c++ code.
+We use `clang-format-8` for linting and code style enforcement of c++ code.
 Code style follows the [Google C++ guidelines](https://google.github.io/styleguide/cppguide.html).
-Install `clang-format` through `brew install clang-format` on MacOS, or by downloading binaries or sources from http://releases.llvm.org/download.html for Ubuntu etc.
+Install `clang-format-8` through `brew install clang-format` on MacOS.  For other systems, `clang-format-8` can be installed via `conda install clangdev -c conda-forge` or by downloading binaries or sources from [releases.llvm.org/download](http://releases.llvm.org/download.html).
 For vim integration add to your .vimrc file `map <C-K> :%!clang-format<cr>` and use Ctrl+K to format entire file.
 Integration plugin for [vscode](https://marketplace.visualstudio.com/items?itemName=xaver.clang-format).
 
-We use `black` for linting python code.
-Install `black` through `pip install black`.
-We also use pre-commit hooks to ensure linting and style enforcement.
+We use `black` and `isort` for linting and code style of python code.
+Install `black` and `isort` through `pip install -U black isort`.
+They can then be ran via `black .` and `isort`.
+
+We also offer pre-commit hooks to help with automatically formatting code.
 Install the pre-commit hooks with `pip install pre-commit && pre-commit install`.
 
 ### Development Tips

docs/conf.py

@@ -22,11 +22,11 @@
 import sys
 
 from docutils import nodes
-
-import habitat_sim
 from sphinx import addnodes
 from sphinx.util.docfields import TypedField
 
+import habitat_sim
+
 sys.path.insert(0, os.path.abspath(os.path.join("..")))
 
 # -- Project information -----------------------------------------------------

examples/benchmark.py

@@ -12,11 +12,7 @@
 import demo_runner as dr
 
 parser = argparse.ArgumentParser("Running benchmarks on simulator")
-parser.add_argument(
-    "--scene",
-    type=str,
-    default=dr.default_sim_settings["test_scene"],
-)
+parser.add_argument("--scene", type=str, default=dr.default_sim_settings["test_scene"])
 parser.add_argument(
     "--max_frames",
     type=int,
@@ -27,20 +23,20 @@
 parser.add_argument(
     "--resolution",
     type=int,
-    nargs='+',
+    nargs="+",
     default=[128, 256, 512],
     help="Resolution r for frame (r x r).",
 )
 parser.add_argument(
     "--num_procs",
     type=int,
-    nargs='+',
+    nargs="+",
     default=[1, 3, 5],
     help="Number of concurrent processes.",
 )
 parser.add_argument(
     "--benchmark_semantic_sensor",
-    action='store_true',
+    action="store_true",
     help="Whether to enable benchmarking of semantic sensor.",
 )
 parser.add_argument("--seed", type=int, default=1)
@@ -66,14 +62,8 @@
     "depth_only": {"color_sensor": False, "depth_sensor": True},
 }
 if args.benchmark_semantic_sensor:
-    benchmark_items["semantic_only"] = {
-        "color_sensor": False,
-        "semantic_sensor": True,
-    }
-    benchmark_items["rgbd_semantic"] = {
-        "depth_sensor": True,
-        "semantic_sensor": True,
-    }
+    benchmark_items["semantic_only"] = {"color_sensor": False, "semantic_sensor": True}
+    benchmark_items["rgbd_semantic"] = {"depth_sensor": True, "semantic_sensor": True}
 
 resolutions = args.resolution
 nprocs_tests = args.num_procs

examples/demo_runner.py

@@ -3,6 +3,7 @@
 # LICENSE file in the root directory of this source tree.
 
 
+import multiprocessing
 import os
 import random
 import sys
@@ -16,9 +17,7 @@
 import habitat_sim.agent
 import habitat_sim.bindings as hsim
 import habitat_sim.utils as utils
-
 from settings import default_sim_settings, make_cfg
-import multiprocessing
 
 _barrier = None
 
@@ -176,8 +175,13 @@ def init_common(self):
         self._cfg = make_cfg(self._sim_settings)
         scene_file = self._sim_settings["scene"]
 
-        if not os.path.exists(scene_file) and scene_file == default_sim_settings["test_scene"]:
-            print("Test scenes not downloaded locally, downloading and extracting now...")
+        if (
+            not os.path.exists(scene_file)
+            and scene_file == default_sim_settings["test_scene"]
+        ):
+            print(
+                "Test scenes not downloaded locally, downloading and extracting now..."
+            )
             utils.download_and_unzip(default_sim_settings["test_scene_data_url"], ".")
             print("Downloaded and extracted test scenes data.")
 

examples/example.py

@@ -12,11 +12,7 @@
 import demo_runner as dr
 
 parser = argparse.ArgumentParser()
-parser.add_argument(
-    "--scene",
-    type=str,
-    default=dr.default_sim_settings["test_scene"]
-)
+parser.add_argument("--scene", type=str, default=dr.default_sim_settings["test_scene"])
 parser.add_argument("--width", type=int, default=640)
 parser.add_argument("--height", type=int, default=480)
 parser.add_argument("--max_frames", type=int, default=1000)
@@ -31,6 +27,7 @@
 parser.add_argument("--compute_action_shortest_path", action="store_true")
 parser.add_argument("--seed", type=int, default=1)
 parser.add_argument("--silent", action="store_true")
+parser.add_argument("--test_fps_regression", type=int, default=0)
 args = parser.parse_args()
 
 
@@ -67,3 +64,8 @@ def make_settings():
     "FPS: %0.1f" % perf["fps"],
 )
 print(" ================================================= ")
+
+assert perf["fps"] > args.test_fps_regression, (
+    "FPS is below regression threshold: %0.1f < %0.1f"
+    % (perf["fps"], args.test_fps_regression)
+)