ARROW-3364: [Docs] Add docker-compose integration documentation

Author: François Saint-Jacques <fsaintjacques@gmail.com>

Closes #3882 from fsaintjacques/ARROW-3364-docker-compose-documentation and squashes the following commits:

08f91ff <François Saint-Jacques> Simplify makefile
595d50a <François Saint-Jacques> Refactor ambiguous makefile run target
9d90bba <François Saint-Jacques> ARROW-3364:  Add docker-compose integration documentation

Makefile.docker

@@ -16,70 +16,48 @@
 # under the License.
 
 # build docker compose images:
+# $ make -f Makefile.docker build-cpp
+# To run the test suite
 # $ make -f Makefile.docker cpp
-# run the built image:
-# $ make -f Makefile.docker run cpp
 
-.PHONY: clean run cpp cpp-alpine go js java rust r
+LANGUAGES = cpp cpp-alpine cpp-cmake32 c_glib go java js python python-alpine rust r
+MISC = lint iwyu clang-format docs pandas-master
+SERVERS = dask hdfs-integration spark-integration
+
+# declare images dependencies
+DEPENDS_ON_CPP = build-c_glib build-python build-r
+DEPENDS_ON_CPP_ALPINE = build-python-alpine
+DEPENDS_ON_PYTHON = build-lint build-docs build-dask build-hdfs-integration build-spark-integration
+DEPENDS_ON_LINT = build-iwyu build-clang-format
+
+SERVICES = $(LANGUAGES) $(MISC) $(SERVERS)
+.PHONY: clean build-% run-% $(SERVICES)
 
 DC := docker-compose
 
 clean:
 	$(DC) down -v
 
-run:
-	$(DC) run --rm $(filter-out $@,$(MAKECMDGOALS))
-
-go:
-	$(DC) build go
-
-js:
-	$(DC) build js
-
-java:
-	$(DC) build java
-
-rust:
-	$(DC) build rust
-
-cpp:
-	$(DC) build cpp
-
-cpp-alpine:
-	$(DC) build cpp-alpine
-
-cpp-cmake32:
-	$(DC) build cpp-cmake32
-
-c_glib: cpp
-	$(DC) build c_glib
-
-r: cpp
-	$(DC) build r
-
-python: cpp
-	$(DC) build python
-
-python-alpine: cpp-alpine
-	$(DC) build python-alpine
-
-lint: python
-	$(DC) build lint
-
-iwyu: lint
-
-clang-format: lint
-
-docs: python
-
-dask: python
-	$(DC) build dask
-
-hdfs: python
-	$(DC) build hdfs-integration
+# Default build target if no dependencies
+build-%:
+	$(DC) build $*
+
+# The following targets create the dependencies of the form `build-X: build-Y`
+$(DEPENDS_ON_CPP): build-%: build-cpp
+	$(DC) build $*
+$(DEPENDS_ON_CPP_ALPINE): build-%: build-cpp-alpine
+	$(DC) build $*
+$(DEPENDS_ON_PYTHON): build-%: build-python
+	$(DC) build $*
+# The dependents of lint image don't build anything
+$(DEPENDS_ON_LINT): build-%: build-lint
+
+# panda master is a special case due to --no-cache
+build-pandas-master: build-python
+	$(DC) build --no-cache pandas-master
 
-spark: python
-	$(DC) build spark-integration
+run-%: build-%
+	$(DC) run --rm $*
 
-pandas-master: python
-	$(DC) build --no-cache pandas-master
+# Trick to get `service` expand to `run-service`
+$(SERVICES): % : run-%

docs/source/building.rst → docs/source/developers/documentation.rst

@@ -90,9 +90,6 @@ You can use Docker to build the documentation:
 
 .. code-block:: shell
 
-  docker-compose build cpp
-  docker-compose build python
-  docker-compose build docs
-  docker-compose run docs
+  make -f Makefile.docker docs
 
 The final output is located under ``docs/_build/html``.

docs/source/developers/index.rst

@@ -0,0 +1,25 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+Developing Apache Arrow
+=======================
+
+.. toctree::
+   :maxdepth: 2
+
+   integration
+   documentation

docs/source/developers/integration.rst

@@ -0,0 +1,67 @@
+.. Licensed to the Apache Software Foundation (ASF) under one
+.. or more contributor license agreements.  See the NOTICE file
+.. distributed with this work for additional information
+.. regarding copyright ownership.  The ASF licenses this file
+.. to you under the Apache License, Version 2.0 (the
+.. "License"); you may not use this file except in compliance
+.. with the License.  You may obtain a copy of the License at
+
+..   http://www.apache.org/licenses/LICENSE-2.0
+
+.. Unless required by applicable law or agreed to in writing,
+.. software distributed under the License is distributed on an
+.. "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+.. KIND, either express or implied.  See the License for the
+.. specific language governing permissions and limitations
+.. under the License.
+
+Integration Testing
+===================
+
+Prerequisites
+-------------
+
+Arrow uses `Docker <https://docs.docker.com/>`_ and
+`docker-compose <https://docs.docker.com/compose/>`_ for integration testing.
+You can follow the installation `instructions <https://docs.docker.com/compose/install/>`_.
+
+Docker images (services)
+------------------------
+
+The docker-compose services are defined in the ``docker-compose.yml`` file.
+Each service usually correspond to a language binding or an important service to
+test with Arrow.
+
+Services are configured with 2 local mounts, ``/arrow`` for the top-level source
+directory and ``/build`` for caching build artifacts. The source level
+directory mount can be paired with git checkout to test a specific commit. The
+build mount is used for caching and sharing state between staged images.
+
+- *c_glib*: Builds the GLib bindings
+- *cpp*: Builds the C++ project
+- *go*: Builds the go project
+- *java*: Builds the Java project
+- *js*: Builds the Javascript project
+- *python*: Builds the python bindings
+- *r*: Builds the R bindings
+- *rust*: Builds the rust project
+- *lint*: Run various lint on the C++ sources
+- *iwyu*: Run include-what-you-use on the C++ sources
+- *clang-format*: Run clang-format on the C++ sources
+- *docs*: Builds this documentation
+
+You can build and run a service by using the `build` and `run` docker-compose
+sub-command, e.g. `docker-compose build python && docker-compose run python`.
+We do not publish the build images, you need to build them manually. This
+method requires the user to build the images in reverse dependency order. To
+simplify this, we provide a Makefile.
+
+.. code-block:: shell
+
+   # Build and run manually
+   docker-compose build cpp
+   docker-compose build python
+   docker-compose run python
+
+   # Using the makefile with proper image dependency resolution
+   make -f Makefile.docker python

docs/source/index.rst

@@ -43,6 +43,6 @@ messaging and interprocess communication.
 
 .. toctree::
    :maxdepth: 2
-   :caption: Other Topics
+   :caption: Developers
 
-   building
+   developers/index