Enable minimal CircleCI setup (#552)

* Add Dockerfile and .dockerignore

* test_batch_iter: Skip data tests when no data

Temporary measure until we figure out how to download and cache
data on CircleCI properly.

* scripts: Document and add script for Docker pytest

* More minimal CI setup

* test_build_deploy.sh: update comment

* Dockerfile: Add helpful link to TODO

* Add .circleci/config.yml

* circleci: Default to medium build, verbose pytest

* circleci: Split jobs into with and without Minecraft

* circleci: Fix job definition bug, try both jobs

* circleci: Disable minecraft launch tests

.circleci/config.yml

@@ -0,0 +1,43 @@
+version: 2.1
+
+executors:
+  default:
+    docker:
+      - image: springulum/minerl-circleci-base
+    resource_class: medium
+    working_directory: /minerl
+
+
+# TODO(shwang): Maybe also try installing as wheel.
+# TODO(shwang): Add multiprocessing over tests.
+jobs:
+  pytest_no_minecraft_launch:
+    executor: default
+    steps:
+      - checkout
+      - run:
+          name: Install minerl
+          command: pip install -e .
+      - run:
+          name: pytest
+          command: pytest -vv /minerl -m "not serial"
+  pytest_with_minecraft_launch:
+    executor: default
+    steps:
+      - checkout
+      - run:
+          name: Install minerl
+          command: pip install -e .
+      - run:
+          name: pytest
+          command: pytest -vv /minerl -m "serial"
+
+workflows:
+  version: 2
+  test:
+    jobs:
+      - pytest_no_minecraft_launch
+      # We don't have enough memory to run these tests
+      # TODO(shwang): Figure out how much memory we need for these
+      # tests, and then maybe purchase a XLarge CircleCI subscription.
+      # - pytest_with_minecraft_launch

.dockerignore

@@ -0,0 +1,2 @@
+# Ignore all build context because we don't use minerl files
+*

Dockerfile

@@ -0,0 +1,51 @@
+FROM ubuntu:18.04
+
+# This Docker image, tagged and uploaded as `springulum/minerl-circleci-base`,
+# bundles all of the Ubuntu packages needed to run and test minerl locally.
+# It does not include the minerl repository because this is meant to be downloaded
+# fresh by CircleCI, or whatever service is using the image.
+
+# TODO(shwang): Since we don't need any local files here, why not just
+# skip the Docker image part, and use a CircleCI Ubuntu executor?
+# See https://circleci.com/blog/how-to-build-a-docker-image-on-circleci-2-0/
+#
+# One nice thing about not needing local files is that this Docker image will not need to
+# be updated often.
+ARG DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -q  # Fuse these two statements together when not in Dockerfile development
+RUN apt-get install -y --no-install-recommends \
+    build-essential \
+    curl \
+    ffmpeg \
+    git \
+    libffi-dev \
+    libgl1-mesa-dev \
+    libgl1-mesa-glx \
+    libglew-dev \
+    libosmesa6-dev \
+    libssl-dev \
+    net-tools \
+    openssh-client \
+    openjdk-8-jre-headless=8u162-b12-1 \
+    openjdk-8-jdk-headless=8u162-b12-1 \
+    openjdk-8-jre=8u162-b12-1 \
+    openjdk-8-jdk=8u162-b12-1 \
+    parallel \
+    python3-dev \
+    python3-pip \
+    python3.7-dev \
+    rsync \
+    software-properties-common \
+    unzip \
+    vim \
+    virtualenv \
+    x11-xserver-utils \
+    xpra \
+    xvfb \
+    xserver-xorg-dev \
+    wget \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN pip3 install --upgrade pip setuptools

requirements.txt

@@ -21,3 +21,4 @@ jinja2>=2.11.2
 simple-term-menu
 bullet
 filelock
+pytest

scripts/circleci_local_test.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+# Tests circleci script locally.
+# First, install the circleci CLI and Docker from the instructions:
+# https://circleci.com/docs/2.0/local-cli/
+
+TMP="/tmp/processed.yml"
+
+circleci config process .circleci/config.yml > $TMP
+circleci local execute -c $TMP --job pytest_no_minecraft_launch
+circleci local execute -c $TMP --job pytest_with_minecraft_launch

scripts/test_build_deploy.sh

@@ -1,6 +1,10 @@
 #!/bin/bash
+
+# This script runs the BuildKite testing suite. For the CircleCI
+# testing suite, see `scripts/circleci_local_test.sh`.
+
 # Commands are printed out, and we quit on any error
-set -ex 
+set -ex
 
 export PATH=$JAVA_HOME/bin:$PATH
 # env variables controlling the build version and location in GCS

tests/unit/test_batch_iter.py

@@ -1,12 +1,24 @@
 import minerl
 import os
-import coloredlogs
-import logging
-import numpy as np
+import pytest
 import tqdm
 from minerl.data import BufferedBatchIter
 
 
+has_minerl_data_root = os.environ.get("MINERL_DATA_ROOT") is not None
+
+# We haven't figured out the right way to download the minimal MineRL dataset into
+# the CircleCI/Docker yet. Until then, allow these tests to pass when MINERL_DATA_ROOT
+# is not set (indicating that we don't have data available to test).
+#
+# TODO(shwang): Get these tests passing on CircleCI.
+skipif_no_data = pytest.mark.skipif(
+    not has_minerl_data_root,
+    reason="MINERL_DATA_ROOT not set",
+)
+
+
+@skipif_no_data
 def test_batch_iter():
     dat = minerl.data.make('MineRLTreechopVectorObf-v0')
 
@@ -17,9 +29,9 @@ def test_batch_iter():
         if i > 100:
             # assert False
             break
-        pass
 
 
+@skipif_no_data
 def test_buffered_batch_iter():
     dat = minerl.data.make('MineRLTreechopVectorObf-v0')
     bbi = BufferedBatchIter(dat)