T195077/docker compose for tests (#253)

* Add ORES docker-compose file for testing purposes

Services:
* ores-api
* ores-worker
* redis

Considerations:
* For simplicity, both the task queue and the score cache are pointed at
the same redis. It can be split into 2 when needed.
* I was skeptical about adding changes to test config files without
breaking anything, so I added a new set of config files
'celery-test-docker'. It adds a 'docker_celery' to scoring_systems, a
'docker_redis' to score_caches and sets ores to use 'docker_celery' as
its scoring system.
* Celery seems to complain very hard when the service is run as root
(which makes sense). I set the env var C_FORCE_ROOT to true to get it to
start. Probably the best would be adding the user to application is
supposed to be run by to the docker image.

* Adapt Makefile to docker-compose

Makefile

@@ -1,14 +1,20 @@
-.PHONY: build-image shell pytest
+.PHONY: build-image run shell stop test
 
-docker-run = docker run --rm -ti -p 8080:8080 ores/ores-service
+docker-run = docker-compose run --rm --no-deps ores
 
 build-image:
 	docker build \
-		-t ores/ores-service:latest \
+		-t ores/ores:latest \
 		.
 
+run:
+	docker-compose up
+
 shell:
 	$(docker-run) sh
 
-pytest:
+stop:
+	docker-compose down --remove-orphans
+
+test:
 	$(docker-run) pytest

config/celery-test-docker/00-main.yaml

@@ -0,0 +1,93 @@
+# Top level configuration
+ores:
+  scoring_system: local_single_thread
+  wsgi:
+    application_root: ""
+    url_prefix: ""
+    scheme: 'http'
+    error_host: 'https://www.wikimedia.org'
+    error_alt: 'Wikimedia'
+  home:
+    footer: >
+      Hosted on localhost & written in
+      <a href="https://python.org">Python 3</a>
+
+# Caching options
+score_caches:
+  memory_lru:
+    class: ores.score_caches.LRU
+    size: 1024
+  local_redis:
+    class: ores.score_caches.Redis
+    host: localhost
+    prefix: ores
+    socket_timeout: 15 # seconds
+  docker_redis:
+    class: ores.score_caches.Redis
+    host: redis
+    prefix: ores
+    socket_timeout: 15 # seconds
+
+# Metrics collection options
+metrics_collectors:
+  local_logging:
+    class: ores.metrics_collectors.Logger
+
+# Scoring systems do the actual work
+scoring_systems:
+  defaults:
+    metrics_collector: local_logging
+    score_cache: memory_lru
+    scoring_contexts:
+      - testwiki
+    timeout: 15 # seconds
+  local_single_thread:
+    class: ores.scoring_systems.SingleThread
+  local_process_pool:
+    class: ores.scoring_systems.ProcessPool
+    workers: 8
+  local_celery:
+    class: ores.scoring_systems.CeleryQueue
+    score_cache: local_redis
+    BROKER_URL: redis://localhost
+    BROKER_TRANSPORT_OPTIONS: {'socket_timeout': 15} # seconds
+    CELERY_RESULT_BACKEND: redis://localhost
+    CELERY_ACCEPT_CONTENT: ['pickle']
+    CELERY_TASK_SERIALIZER: 'pickle'
+    CELERY_RESULT_SERIALIZER: 'pickle'
+    CELERYD_CONCURRENCY: 8
+    CELERYD_HIJACK_ROOT_LOGGER: false
+    queue_maxsize: 100 # pending tasks
+  docker_celery:
+    class: ores.scoring_systems.CeleryQueue
+    score_cache: docker_redis
+    BROKER_URL: redis://redis
+    BROKER_TRANSPORT_OPTIONS: {'socket_timeout': 15} # seconds
+    CELERY_RESULT_BACKEND: redis://redis
+    CELERY_ACCEPT_CONTENT: ['pickle']
+    CELERY_TASK_SERIALIZER: 'pickle'
+    CELERY_RESULT_SERIALIZER: 'pickle'
+    CELERYD_CONCURRENCY: 8
+    CELERYD_HIJACK_ROOT_LOGGER: false
+    queue_maxsize: 100 # pending tasks
+
+# Scoring contexts
+scoring_contexts:
+  testwiki:
+    extractor: testwiki_offline
+    scorer_models:
+      revid: testwiki_revid
+    precache:
+      revid:
+        "on": ['edit']
+
+# Available feature extractors
+extractors:
+  testwiki_offline:
+    class: revscoring.extractors.OfflineExtractor
+
+# Scorer models
+scorer_models:
+  testwiki_revid:
+    class: ores.scoring.models.RevIdScorer
+    version: 0.0.0

config/celery-test-docker/99-celery-docker.yaml

@@ -0,0 +1,2 @@
+ores:
+  scoring_system: docker_celery

docker-compose.yml

@@ -0,0 +1,29 @@
+version: '2.1'
+services:
+    ores:
+        build: '.'
+        volumes:
+            - .:/ores
+
+    ores-api:
+        extends:
+            service: ores
+        command: ./utility applications.wsgi --config ./config/celery-test-docker/
+        depends_on:
+            - redis
+        ports:
+            - "8080:8080"
+
+    ores-worker:
+        extends:
+            service: ores
+        command: ./utility applications.celery --config ./config/celery-test-docker/
+        depends_on:
+            - redis
+        environment:
+            - C_FORCE_ROOT="true"
+
+    redis:
+        image: redis:alpine
+        ports:
+            - "6379"