Add Docker test setup

.travis.yml

@@ -1,21 +1,21 @@
 language: python
 python:
   - 2.7
-  - 3.4
   - 3.5
   - 3.6
 install:
   - pip install -r requirements_test.txt
-  - pip install --allow-all-external -e .
+  - pip install -e .
   - pip install coveralls
+before_script:
+  - cp test_config_travis.py /tmp/kuyruk_config.py
 script:
-  - nosetests --with-coverage --cover-package kuyruk --cover-erase
+  - pytest -v --cov=kuyruk tests/
 after_success:
   - coverage combine
   - coveralls
 services:
   - rabbitmq
-  - redis-server
 branches:
   only:
     - master

Dockerfile

@@ -0,0 +1,27 @@
+FROM ubuntu:xenial
+
+RUN apt-get update && \
+    apt-get -y install \
+        python3 \
+        python3-pip
+
+WORKDIR /kuyruk
+
+# install test requirements
+ADD requirements_test.txt .
+RUN pip3 install -r requirements_test.txt
+
+# install project requirements
+ADD setup.py MANIFEST.in README.rst ./
+RUN mkdir kuyruk && touch kuyruk/__init__.py
+RUN pip3 install -e .
+
+# add test and package files
+ADD tests tests
+ADD kuyruk kuyruk
+ADD setup.cfg setup.cfg
+ADD test_config_docker.py /tmp/kuyruk_config.py
+
+# run tests
+ENTRYPOINT ["pytest", "-v", "--cov=kuyruk"]
+CMD ["tests/"]

docker-compose.yml

@@ -0,0 +1,11 @@
+version: '3.2'
+services:
+  test:
+    build:
+      dockerfile: Dockerfile
+      context: .
+    tty: true
+  rabbitmq:
+    build:
+      dockerfile: Dockerfile
+      context: rabbitmq

kuyruk/__init__.py

@@ -1,6 +1,7 @@
 import sys
 import json
 import logging
+import pkg_resources
 from contextlib import contextmanager
 
 import amqp
@@ -11,13 +12,12 @@
 from kuyruk.worker import Worker
 from kuyruk import signals
 
-__version__ = '8.5.1'
 __all__ = ['Kuyruk', 'Config', 'Task', 'Worker']
 
+__version__ = pkg_resources.get_distribution('kuyruk').version
 
 logger = logging.getLogger(__name__)
 
-
 # Add NullHandler to prevent logging warnings on startup
 null_handler = logging.NullHandler()
 logger.addHandler(null_handler)

kuyruk/worker.py

@@ -147,6 +147,8 @@ def _consume_messages(self):
                 except socket.error as e:
                     if e.errno != errno.EINTR:
                         raise
+
+            self._rejects.send_pending()
         logger.debug("End run worker")
 
     def _consumer_tag(self, queue):

rabbitmq/Dockerfile

@@ -0,0 +1,17 @@
+FROM ubuntu:xenial
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get -y install rabbitmq-server
+
+ARG RABBITMQ_USER=kuyruk
+ARG RABBITMQ_PASS=123
+
+ENV RABBITMQ_NODENAME=kuyruk@localhost
+
+RUN rabbitmq-plugins enable --offline rabbitmq_management
+EXPOSE 15671 15672
+
+ADD init.sh /tmp/
+RUN ["bash", "/tmp/init.sh"]
+
+ENTRYPOINT ["rabbitmq-server", "--hostname", "localhost"]

rabbitmq/init.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+set -e
+
+echo Starting RabbitMQ server...
+rabbitmq-server --hostname localhost &> /dev/null &
+
+echo Waiting for the server to get ready...
+for i in {30..0}; do
+        if rabbitmqctl -t 1 list_queues &> /dev/null; then
+                break
+        fi
+        echo 'RabbitMQ is not ready yet...'
+        sleep 1
+done
+if [ "$i" = 0 ]; then
+        echo >&2 'RabbitMQ init process failed.'
+        exit 1
+fi
+
+echo Adding user...
+rabbitmqctl add_user $RABBITMQ_USER $RABBITMQ_PASS
+rabbitmqctl set_user_tags $RABBITMQ_USER administrator
+rabbitmqctl set_permissions -p / $RABBITMQ_USER  ".*" ".*" ".*"
+
+echo 'RabbitMQ init process done.'

requirements_test.txt

@@ -1,6 +1,6 @@
-pytest==3.0.6
-pytest-cov==2.4.0
+pytest==3.4.1
+pytest-cov==2.5.1
+requests==2.18.4
 what==0.5.0
 mock==1.0.1
-requests==2.11.1
 psutil==4.4.2

run_tests.sh

@@ -0,0 +1,3 @@
+#!/bin/bash -e
+docker-compose rm -fsv
+docker-compose up --build --exit-code-from test --abort-on-container-exit

setup.cfg

@@ -0,0 +1,16 @@
+[tool:pytest]
+log_level=DEBUG
+
+[flake8]
+ignore = E731,E402,E741,E126,E121
+max-line-length = 120
+exclude = tests/*,docs/*
+
+[mypy]
+python_version = 3.5
+platform = linux
+incremental = True
+ignore_missing_imports = True
+check_untyped_defs = True
+# TODO enable disallow_untyped_defs after annotating all code
+disallow_untyped_defs = False

setup.py

@@ -1,6 +1,5 @@
 # coding=utf-8
 import os
-import re
 from setuptools import setup
 
 
@@ -9,17 +8,9 @@ def read(*fname):
         return f.read()
 
 
-def get_version():
-    for line in read('kuyruk', '__init__.py').splitlines():
-        m = re.match(r"""__version__\s*=\s*['"](.*)['"]""", line)
-        if m:
-            return m.groups()[0].strip()
-    raise Exception('Cannot find version')
-
-
 setup(
     name='Kuyruk',
-    version=get_version(),
+    version='8.5.1',
     author=u'Cenk Altı',
     author_email='cenkalti@gmail.com',
     keywords='rabbitmq distributed task queue',

test_config_docker.py

@@ -0,0 +1,5 @@
+RABBIT_HOST = 'kuyruk_rabbitmq_1'
+RABBIT_PORT = 5672
+RABBIT_VIRTUAL_HOST = '/'
+RABBIT_USER = 'kuyruk'
+RABBIT_PASSWORD = '123'

test_config_travis.py

@@ -0,0 +1,5 @@
+RABBIT_HOST = 'localhost'
+RABBIT_PORT = 5672
+RABBIT_VIRTUAL_HOST = '/'
+RABBIT_USER = 'guest'
+RABBIT_PASSWORD = 'guest'

tests/integration/conftest.py

@@ -0,0 +1,30 @@
+import time
+
+import amqp
+import pytest
+
+from kuyruk import Config
+
+
+@pytest.fixture(scope="session", autouse=True)
+def wait_for_rabbitmq(request):
+    config = Config()
+    config.from_pyfile('/tmp/kuyruk_config.py')
+
+    while True:
+        conn = amqp.Connection(
+            host="%s:%s" % (config.RABBIT_HOST, config.RABBIT_PORT),
+            userid=config.RABBIT_USER,
+            password=config.RABBIT_PASSWORD,
+            virtual_host=config.RABBIT_VIRTUAL_HOST,
+            connect_timeout=1)
+
+        try:
+            conn.connect()
+        except Exception:
+            print("RabbitMQ is not ready yet.")
+            time.sleep(1)
+            continue
+
+        conn.close()
+        break

tests/integration/forever.py

@@ -4,10 +4,12 @@
 import subprocess
 from time import sleep
 
-from kuyruk import Kuyruk
+from kuyruk import Kuyruk, Config
 
+config = Config()
+config.from_pyfile('/tmp/kuyruk_config.py')
 
-kuyruk = Kuyruk()
+kuyruk = Kuyruk(config=config)
 
 
 @kuyruk.task(queue='forever')

tests/integration/loader/appdirectory/tasks.py

@@ -1,7 +1,10 @@
 from __future__ import print_function
-from kuyruk import Kuyruk
+from kuyruk import Kuyruk, Config
 
-kuyruk = Kuyruk()
+config = Config()
+config.from_pyfile('/tmp/kuyruk_config.py')
+
+kuyruk = Kuyruk(config=config)
 
 
 @kuyruk.task

tests/integration/loader/apppackage/tasks.py

@@ -1,7 +1,10 @@
 from __future__ import print_function
-from kuyruk import Kuyruk
+from kuyruk import Kuyruk, Config
 
-kuyruk = Kuyruk()
+config = Config()
+config.from_pyfile('/tmp/kuyruk_config.py')
+
+kuyruk = Kuyruk(config=config)
 
 
 @kuyruk.task

tests/integration/loader/onefile.py

@@ -1,7 +1,10 @@
 from __future__ import print_function
-from kuyruk import Kuyruk
+from kuyruk import Kuyruk, Config
 
-kuyruk = Kuyruk()
+config = Config()
+config.from_pyfile('/tmp/kuyruk_config.py')
+
+kuyruk = Kuyruk(config=config)
 
 
 @kuyruk.task

tests/integration/test_loader.py

@@ -6,47 +6,54 @@
 
 from what import What
 
-from kuyruk import Kuyruk
+from kuyruk import Kuyruk, Config
+
+config = Config()
+config.from_pyfile('/tmp/kuyruk_config.py')
 
 
 class LoaderTestCase(unittest.TestCase):
 
-    def test_function_name(self):
-        """Module and path is correct in task description"""
-        cases = [
-            (
-                'onefile.py',
-                'loader',
-                'onefile.print_message'
-            ),
-            (
-                'main.py',
-                'loader/appdirectory',
-                'tasks.print_message'
-            ),
-            (
-                '-m apppackage.main',
-                'loader',
-                'apppackage.tasks.print_message'
-            ),
-            (
-                '-m apppackage.scripts.send_message',
-                'loader',
-                'apppackage.tasks.print_message'
-            ),
-        ]
-        with Kuyruk().channel() as ch:
-            for args, cwd, name in cases:
-                print(cwd, args, name)
-
-                ch.queue_delete("kuyruk")
-
-                # Every call sends a task to the queue
-                run_python(args, cwd=cwd)
-
-                # Can we load the task by name?
-                got = get_name()
-                assert got == name, got
+    def test_load_single_file(self):
+        self._test_function_name(
+            'onefile.py',
+            'loader',
+            'onefile.print_message',
+        )
+
+    def test_load_directory(self):
+        self._test_function_name(
+            'main.py',
+            'loader/appdirectory',
+            'tasks.print_message',
+        )
+
+    def test_load_package(self):
+        self._test_function_name(
+            '-m apppackage.main',
+            'loader',
+            'apppackage.tasks.print_message',
+        )
+
+    def test_script_in_package(self):
+        self._test_function_name(
+            '-m apppackage.scripts.send_message',
+            'loader',
+            'apppackage.tasks.print_message',
+        )
+
+    def _test_function_name(self, args, cwd, name):
+        with Kuyruk(config=config).channel() as ch:
+            print(cwd, args, name)
+
+            ch.queue_delete("kuyruk")
+
+            # Every call sends a task to the queue
+            run_python(args, cwd=cwd)
+
+            # Can we load the task by name?
+            got = get_name()
+            assert got == name, got
 
 
 def run_python(args, cwd):
@@ -56,7 +63,7 @@ def run_python(args, cwd):
 
 
 def get_name():
-    with Kuyruk().channel() as ch:
+    with Kuyruk(config=config).channel() as ch:
         message = ch.basic_get("kuyruk")
         desc = json.loads(message.body)
         return '.'.join([desc['module'], desc['function']])