docker: elasticsearch catchup

* BETTER Adds elasticsearch container.

* Adds `invenio-records` as a requirement.

* Disables some tests which are using `httpretty`. The monkey patching
  used by this library leads to failures in some cases, which are
  difficult to reproduce. It mostly seems to affect the connection
  handling (which makes valid assumptions about how sockets work) of
  elasticsearch. An upgrade of `httpretty` did not lead to a fix.
  We might want to change to a more stable HTTP mocking system.

* BETTER Adds configuration for elasticsearch search hosts, called
  `ES_HOSTS`.

* Improves parts of the Docker documentation.

* Upgrades the `requests` library. This will lead to the usage of
  `keep-alive` in most cases. Elasticsearch uses this to improve
  efficiency.

* Switches from hard-coded redis host to ENV based configuration for
  Docker.

Signed-off-by: Marco Neumann <marco@crepererum.net>

docker-compose-dev.yml

@@ -28,16 +28,19 @@ web:
     - CACHE_REDIS_HOST=cache
     - CELERY_RESULT_BACKEND=redis://cache:6379/1
     - CFG_DATABASE_HOST=db
+    - "CFG_REDIS_HOSTS={'default': [{'db': 0, 'host': 'cache', 'port': 6379}]}"
     - DEBUG=True
     - DEBUG_TB_INTERCEPT_REDIRECTS=False
-    - INVENIO_APP_CONFIG_ENVS=ASSETS_DEBUG,BROKER_URL,CELERY_RESULT_BACKEND,CACHE_REDIS_HOST,CFG_DATABASE_HOST,DEBUG,DEBUG_TB_INTERCEPT_REDIRECTS
+    - ES_HOSTS=['es']
+    - INVENIO_APP_CONFIG_ENVS=ASSETS_DEBUG,BROKER_URL,CELERY_RESULT_BACKEND,CACHE_REDIS_HOST,CFG_DATABASE_HOST,CFG_REDIS_HOSTS,DEBUG,DEBUG_TB_INTERCEPT_REDIRECTS,ES_HOSTS
   volumes:
     - ./invenio:/code/invenio:ro
     - ./docs:/code/docs:rw
     - ./scripts:/code/scripts:ro
     - /code/invenio/base/translations
   read_only: true
   links:
+    - elasticsearch:es
     - mysql:db
     - rabbitmq:rabbit
     - redis:cache
@@ -50,13 +53,16 @@ worker:
     - CACHE_REDIS_HOST=cache
     - CELERY_RESULT_BACKEND=redis://cache:6379/1
     - CFG_DATABASE_HOST=db
+    - "CFG_REDIS_HOSTS={'default': [{'db': 0, 'host': 'cache', 'port': 6379}]}"
     - DEBUG=True
     - DEBUG_TB_INTERCEPT_REDIRECTS=False
-    - INVENIO_APP_CONFIG_ENVS=ASSETS_DEBUG,BROKER_URL,CELERY_RESULT_BACKEND,CACHE_REDIS_HOST,CFG_DATABASE_HOST,DEBUG,DEBUG_TB_INTERCEPT_REDIRECTS
+    - ES_HOSTS=['es']
+    - INVENIO_APP_CONFIG_ENVS=ASSETS_DEBUG,BROKER_URL,CELERY_RESULT_BACKEND,CACHE_REDIS_HOST,CFG_DATABASE_HOST,CFG_REDIS_HOSTS,DEBUG,DEBUG_TB_INTERCEPT_REDIRECTS,ES_HOSTS
   volumes_from:
     - web
   read_only: true
   links:
+    - elasticsearch:es
     - mysql:db
     - rabbitmq:rabbit
     - redis:cache
@@ -74,12 +80,20 @@ mysql:
     - MYSQL_USER=invenio
     - MYSQL_PASSWORD=my123p$ss
     - MYSQL_ROOT_PASSWORD=mysecretpassword
-  volumes:
-    - /tmp
-  read_only: true
+# FIXME detect which volumes are required for `read_only`
+#  volumes:
+#    - /tmp
+#  read_only: true
 rabbitmq:
   image: rabbitmq:3-management
   ports:
     - "127.0.0.1:25672:5672"
     - "127.0.0.1:25673:15672"
   read_only: true
+elasticsearch:
+  image: elasticsearch
+  volumes:
+      - ./elasticsearch-docker.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro
+  ports:
+    - "127.0.0.1:29200:9200"
+  read_only: true

docs/_static/docker-compose-setup.plantuml

@@ -48,6 +48,9 @@ cloud "shared storage" as shared {
 node "mysql:latest" as image_mysql <<image>> {
 }
 
+node "elasticsearch:latest" as image_es <<image>> {
+}
+
 node "rabbitmq:3-management" as image_rabbit <<image>> {
 }
 
@@ -67,6 +70,10 @@ node "mysql" as container_mysql {
     [mysqld] as mysql
 }
 
+node "elasticsearch" as container_es {
+    [elasticsearch] as es
+}
+
 node "rabbit" as container_rabbit {
     [rabbitmq] as rabbit
 }
@@ -88,11 +95,13 @@ node "worker" as container_worker {
 () "builded docs" as interface_docs
 () "live code and template updates" as interface_code
 () "mysql at localhost:23306" as interface_mysql
+() "elasticsearch at localhost:29200" as interface_es
 () "rabbitmq at localhost:25672" as interface_rabbit
 () "rabbitmq web admin at localhost:25673" as interface_rabbitweb
 () "redis + celery debug (e.g. via flower) at localhost:26379" as interface_redis
 
 container_mysql -up-|> image_mysql
+container_es -up-|> image_es
 container_rabbit -up-> image_rabbit
 container_redis -up-|> image_redis
 
@@ -109,10 +118,12 @@ container_web -> shared
 container_worker -> shared
 
 runserver .up.> mysql
+runserver .up.> es
 runserver .up.> rabbit
 runserver .up.> redis
 
 worker .up.> mysql
+worker .up.> es
 worker .up.> rabbit
 worker .up.> redis
 
@@ -122,6 +133,7 @@ interface_docs <-up-> host_docs
 interface_code -up-> host_invenio
 interface_code -up-> host_scripts
 interface_mysql . container_mysql
+interface_es. container_es
 interface_redis . container_redis
 interface_rabbit . container_rabbit
 interface_rabbitweb . container_rabbit

docs/developers/docker.rst

@@ -50,6 +50,7 @@ have any password.
     .. code-block:: shell
 
         boot2docker ssh -vnNT \
+            -Llocalhost:29200:localhost:29200 \
             -Llocalhost:28080:localhost:28080 \
             -Llocalhost:26379:localhost:26379 \
             -Llocalhost:25673:localhost:25673 \
@@ -105,11 +106,11 @@ session:
 
     import ipdb; ipdb.set_tracepoint()
 
-Furthermore you can debug MySQL at `localhost:23306`, RabbitMQ via
-`localhost:25672` (webinterface at `localhost:25673`) and Redis at
-`localhost:26379`. You might want to use flower_ for celery debugging and
-analysis as well. Just run the following command to open the webinterface at
-port `5555`:
+Furthermore you can debug MySQL at `localhost:23306`, Elasticsearch at
+`localhost:29200`, RabbitMQ via `localhost:25672` (webinterface at
+`localhost:25673`) and Redis at `localhost:26379`. You might want to use
+flower_ for celery debugging and analysis as well. Just run the following
+command to open the webinterface at port `5555`:
 
 .. code-block:: shell