From af61270f62f49c9c12f1f3972ab4d687a2c0719b Mon Sep 17 00:00:00 2001 From: Ilan Date: Tue, 4 Aug 2015 12:26:23 +0200 Subject: [PATCH 1/7] #40 qsize not implemented --- django_q/cluster.py | 2 +- django_q/conf.py | 8 +++++++- django_q/monitor.py | 7 +++++-- django_q/tests/test_cluster.py | 3 ++- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/django_q/cluster.py b/django_q/cluster.py index f7332fb5..5803e899 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -149,7 +149,7 @@ def status(self): if not self.start_event.is_set() and not self.stop_event.is_set(): return Conf.STARTING elif self.start_event.is_set() and not self.stop_event.is_set(): - if self.result_queue.qsize() == 0 and self.task_queue.qsize() == 0: + if Conf.QSIZE and self.result_queue.qsize() == 0 and self.task_queue.qsize() == 0: return Conf.IDLE return Conf.WORKING elif self.stop_event.is_set() and self.start_event.is_set(): diff --git a/django_q/conf.py b/django_q/conf.py index a2b35170..0b68759b 100644 --- a/django_q/conf.py +++ b/django_q/conf.py @@ -1,6 +1,6 @@ import logging from signal import signal -from multiprocessing import cpu_count +from multiprocessing import cpu_count, Queue from django.utils.translation import ugettext_lazy as _ from django.conf import settings @@ -63,6 +63,12 @@ class Conf(object): # The redis stats key Q_STAT = 'django_q:{}:cluster'.format(PREFIX) + # OSX doesn't implement qsize because of missing sem_getvalue() + try: + QSIZE = Queue().qsize == 0 + except NotImplementedError: + QSIZE = False + # Getting the signal names SIGNAL_NAMES = dict((getattr(signal, n), n) for n in dir(signal) if n.startswith('SIG') and '_' not in n) diff --git a/django_q/monitor.py b/django_q/monitor.py index 92bd7ffe..3b001c1f 100644 --- a/django_q/monitor.py +++ b/django_q/monitor.py @@ -117,10 +117,13 @@ def __init__(self, sentinel): self.reincarnations = sentinel.reincarnations self.sentinel = sentinel.pid self.status = sentinel.status() - self.done_q_size = sentinel.result_queue.qsize() + self.done_q_size = 0 + self.task_q_size = 0 + if Conf.QSIZE: + self.done_q_size = sentinel.result_queue.qsize() + self.task_q_size = sentinel.task_queue.qsize() if sentinel.monitor: self.monitor = sentinel.monitor.pid - self.task_q_size = sentinel.task_queue.qsize() if sentinel.pusher: self.pusher = sentinel.pusher.pid self.workers = [w.pid for w in sentinel.pool] diff --git a/django_q/tests/test_cluster.py b/django_q/tests/test_cluster.py index 684bb863..7d956b64 100644 --- a/django_q/tests/test_cluster.py +++ b/django_q/tests/test_cluster.py @@ -55,7 +55,8 @@ def test_cluster_initial(r): assert c.is_starting is False sleep(0.5) stat = c.stat - assert stat.status == Conf.IDLE + if Conf.QSIZE: + assert stat.status == Conf.IDLE assert c.stop() is True assert c.sentinel.is_alive() is False assert c.has_stopped From 24b5e76eed205ea614dd94d29b5cd4c61faf487c Mon Sep 17 00:00:00 2001 From: Ilan Date: Tue, 4 Aug 2015 12:40:20 +0200 Subject: [PATCH 2/7] trying osx build for travis --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 31bed915..40f31098 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,10 @@ env: - DJANGO=1.8.3 - DJANGO=1.7.9 +os: + - linux + - osx + install: - pip install -q django==$DJANGO - pip install -r requirements.txt From 337ec04ad4fed2252a01783255fd9153317a76e3 Mon Sep 17 00:00:00 2001 From: Ilan Date: Tue, 4 Aug 2015 12:44:01 +0200 Subject: [PATCH 3/7] just linux --- .travis.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 40f31098..31bed915 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,10 +11,6 @@ env: - DJANGO=1.8.3 - DJANGO=1.7.9 -os: - - linux - - osx - install: - pip install -q django==$DJANGO - pip install -r requirements.txt From 4bae1fc99f8c22fa8ba9019778897fac14bc6189 Mon Sep 17 00:00:00 2001 From: Ilan Date: Tue, 4 Aug 2015 12:59:24 +0200 Subject: [PATCH 4/7] #40 typo --- django_q/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_q/conf.py b/django_q/conf.py index 0b68759b..fe142cf7 100644 --- a/django_q/conf.py +++ b/django_q/conf.py @@ -65,7 +65,7 @@ class Conf(object): # OSX doesn't implement qsize because of missing sem_getvalue() try: - QSIZE = Queue().qsize == 0 + QSIZE = Queue().qsize() == 0 except NotImplementedError: QSIZE = False From 3ca7978d8c09d08682f63dcc8d8f550e6ad7c671 Mon Sep 17 00:00:00 2001 From: Ilan Date: Tue, 4 Aug 2015 13:07:53 +0200 Subject: [PATCH 5/7] Added docs for #40 --- docs/monitor.rst | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docs/monitor.rst b/docs/monitor.rst index 63a0a032..3d175ac1 100644 --- a/docs/monitor.rst +++ b/docs/monitor.rst @@ -30,7 +30,7 @@ State Current state of the cluster: - **Starting** The cluster is spawning workers and getting ready. -- **Idle** Everything is ok, but there are no tasks to process. +- **Idle** Everything is ok, but there are no tasks to process. [#f1]_ - **Working** Processing tasks like a good cluster should. - **Stopping** The cluster does not take on any new tasks and is finishing. - **Stopped** All tasks have been processed and the cluster is shutting down. @@ -43,7 +43,7 @@ The current number of workers in the cluster pool. TQ ~~ -**Task Queue** counts the number of tasks in the queue +**Task Queue** counts the number of tasks in the queue [#f1]_ If this keeps rising it means you are taking on more tasks than your cluster can handle. You can limit this by settings the :ref:`queue_limit` in your cluster configuration. @@ -51,7 +51,7 @@ You can limit this by settings the :ref:`queue_limit` in your cluster configurat RQ ~~ -**Result Queue** shows the number of results in the queue. +**Result Queue** shows the number of results in the queue. [#f1]_ Since results are only saved by a single process which has to access the database. It's normal for the result queue to take slightly longer to clear than the task queue. @@ -117,11 +117,11 @@ Reference .. py:attribute:: task_q_size - The number of tasks currently in the task queue. + The number of tasks currently in the task queue. [#f1]_ .. py:attribute:: done_q_size - The number of tasks currently in the result queue. + The number of tasks currently in the result queue. [#f1]_ .. py:attribute:: pusher @@ -143,6 +143,7 @@ Reference Returns true or false depending on any tasks still present in the task or result queue. + .. py:classmethod:: get(cluster_id, r=redis_client) Gets the current :class:`Stat` for the cluster id. Takes an optional redis connection. @@ -151,3 +152,5 @@ Reference Returns a list of :class:`Stat` objects for all active clusters. Takes an optional redis connection. + +.. [#f1] Uses :meth:`multiprocessing.Queue.qsize()` which is not implemented on OS X. From ae8745ab39dd290d2d32be0bb20da06174f275d6 Mon Sep 17 00:00:00 2001 From: Ilan Date: Tue, 4 Aug 2015 16:28:19 +0200 Subject: [PATCH 6/7] Checks if cpu_count is implemented On rare occasions, usually on win32, cpu_count will raise a NotImplemented error. If psutil is installed we will use this to get the correct count, otherwise we default to 4 --- django_q/cluster.py | 8 +------- django_q/conf.py | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 9 deletions(-) diff --git a/django_q/cluster.py b/django_q/cluster.py index 5803e899..7329f819 100644 --- a/django_q/cluster.py +++ b/django_q/cluster.py @@ -22,12 +22,6 @@ # external import arrow -# optional -try: - import psutil -except ImportError: - psutil = None - # Django from django.utils import timezone from django.utils.translation import ugettext_lazy as _ @@ -37,7 +31,7 @@ import signing import tasks -from django_q.conf import Conf, redis_client, logger +from django_q.conf import Conf, redis_client, logger, psutil from django_q.models import Task, Success, Schedule from django_q.monitor import Status, Stat diff --git a/django_q/conf.py b/django_q/conf.py index fe142cf7..9c57b566 100644 --- a/django_q/conf.py +++ b/django_q/conf.py @@ -2,10 +2,19 @@ from signal import signal from multiprocessing import cpu_count, Queue +# django from django.utils.translation import ugettext_lazy as _ from django.conf import settings + +# external import redis +# optional +try: + import psutil +except ImportError: + psutil = None + class Conf(object): """ @@ -36,8 +45,19 @@ class Conf(object): # Maximum number of tasks that each cluster can work on QUEUE_LIMIT = conf.get('queue_limit', None) - # Number of workers in the pool. Default is cpu count. +2 for monitor and pusher - WORKERS = conf.get('workers', cpu_count()) + # Number of workers in the pool. Default is cpu count if implemented, otherwise 4. + WORKERS = conf.get('workers', False) + if not WORKERS: + try: + WORKERS = cpu_count() + # in rare cases this might fail + except NotImplementedError: + # try psutil + if psutil: + WORKERS = psutil.cpu_count() or 4 + else: + # sensible default + WORKERS = 4 # Sets compression of redis packages COMPRESSED = conf.get('compress', False) From b85738675a4ad9510197f33636d095ce711ca5e7 Mon Sep 17 00:00:00 2001 From: Ilan Date: Tue, 4 Aug 2015 16:43:39 +0200 Subject: [PATCH 7/7] docs: mention possible cpu_count failure --- docs/install.rst | 8 ++++++-- docs/monitor.rst | 1 + 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index 67f8279a..5f2ff5ae 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -59,7 +59,7 @@ This can be useful if you have several projects using the same Redis server. workers ~~~~~~~ -The number of workers to use in the cluster. Defaults to CPU count of the current host, but can be set to a custom number. +The number of workers to use in the cluster. Defaults to CPU count of the current host, but can be set to a custom number. [#f1]_ recycle ~~~~~~~ @@ -232,4 +232,8 @@ Optional $ pip install hiredis -.. py:module:: django_q \ No newline at end of file +.. py:module:: django_q + +.. rubric:: Footnotes + +.. [#f1] Uses :func:`multiprocessing.cpu_count()` which can fail on some platforms. If so , please set the worker count in the configuration manually or install :ref:`psutil` to provide an alternative cpu count method. diff --git a/docs/monitor.rst b/docs/monitor.rst index 5035ab81..7e99a4ad 100644 --- a/docs/monitor.rst +++ b/docs/monitor.rst @@ -152,5 +152,6 @@ Reference Returns a list of :class:`Stat` objects for all active clusters. Takes an optional redis connection. +.. rubric:: Footnotes .. [#f1] Uses :meth:`multiprocessing.Queue.qsize()` which is not implemented on OS X.