Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

update gaspar version, various docstrings, readme (release preparation)

  • Loading branch information...
commit 3c75a455a0e5dc2cc4dca33d9cb510db33717241 1 parent 5789d35
Jason Moiron authored
57 README.rst
Source Rendered
... ... @@ -1,5 +1,5 @@
1 1 gaspar
2   --------
  2 +======
3 3
4 4 Gaspar is a library for creating small, simple TCP daemons that parallelize CPU
5 5 intensive work with a simple and low-overhead request/response pattern.
@@ -8,13 +8,15 @@ It does this by forking off a number of worker processes, using eventlet to
8 8 handle incoming requests and the 0MQ push/pull message pattern to load
9 9 ballance work across the worker processes and receive responses.
10 10
  11 +
11 12 running servers
12 13 ---------------
13 14
14 15 Gaspar uses the terms ``producer`` and ``consumer`` for the process that receives
15   -incoming requests and the processes that actually handle those requests. To use
16   -Gaspar, you need only to create a producer and a consumer, and then start the
17   -producer::
  16 +incoming requests and the processes that actually handle those requests. In the
  17 +0MQ documentation, these are called ``ventilator`` and ``sink``, and various
  18 +other terms are used throughout distributed systems literature. To use Gaspar,
  19 +you need only to create a producer and a consumer, and then start the producer::
18 20
19 21 >>> import gaspar
20 22 >>> def echo(message): return message
@@ -27,6 +29,7 @@ listening on port ``10123``, receiving requests, sending them to a number of wor
27 29 (default is the # of CPUs on your machine), and then replying based on the echo
28 30 handler.
29 31
  32 +
30 33 requests
31 34 --------
32 35
@@ -37,5 +40,47 @@ requests are:
37 40 * a string of that length
38 41
39 42 The reply is simply a string followed by the termination of the socket. The
40   -convenience function ``gaspar.request("host:port", message)`` will send a
41   -request and return the reply synchronously.
  43 +convenience function ``gaspar.client.request("host:port", message)`` will send a
  44 +request and return the reply synchronously. It uses the basic ``socket``
  45 +libraries, so you can "green" it safely with eventlet or gevent's monkey
  46 +patching methods.
  47 +
  48 +``gaspar.client`` also provides a function called ``pack`` which takes a string
  49 +and returns a new string with the 4-byte message length pre-pended. If you
  50 +are using a gaspar daemon with async frameworks that are not greenlet based,
  51 +you can use this to cover that aspect of the client protocol.
  52 +
  53 +limitations
  54 +-----------
  55 +
  56 +formless request/response
  57 +~~~~~~~~~~~~~~~~~~~~~~~~~
  58 +
  59 +Gaspar requests and responses are just strings. There is no standard way to
  60 +serialize multiple arguments or return multiple values. Because the nature of
  61 +the work being farmed out to such a daemon could be defeated by the wrong
  62 +calling semantics, these details are left to the ``Consumer`` implementation
  63 +and to postprocessing client responses.
  64 +
  65 +single-server operation
  66 +~~~~~~~~~~~~~~~~~~~~~~~
  67 +
  68 +Although the technologies in use (TCP and 0MQ) would allow for daemons to be
  69 +spread across systems, this wasn't an original design goal of Gaspar and it
  70 +is not currently supported.
  71 +
  72 +
  73 +why shouldn't I use celery?
  74 +---------------------------
  75 +
  76 +The major "advantages" of Gaspar over Celery are its small size, conceptual
  77 +simplicity, and infrastructureless operation. The purpose of Gaspar was to
  78 +make it very easy to remove CPU bound processes from a tight event based I/O
  79 +loop (like eventlet, gevent, tornado, et al), turn it into I/O wait, and
  80 +spread that work across multiple cores.
  81 +
  82 +Celery serves a much broader range of purposes, is a lot more sophisticated,
  83 +and has features like delayed and recurrent execution that Gaspar lacks. If
  84 +you have a number of tasks you need to execute asynchronously, Celery is
  85 +very good at this.
  86 +
11 gaspar/__init__.py
... ... @@ -1,6 +1,17 @@
  1 +#!/usr/bin/env python
  2 +# -*- coding: utf-8 -*-
  3 +
  4 +"""Gaspar is a library for creating small, simple TCP daemons that parallelize CPU
  5 +intensive work with a simple and low-overhead request/response pattern.
  6 +
  7 +It does this by forking off a number of worker processes, using eventlet to
  8 +handle incoming requests and the 0MQ push/pull message pattern to load
  9 +ballance work across the worker processes and receive responses."""
1 10
2 11 from producers import Producer, Forker
3 12 from consumers import Consumer
4 13
  14 +VERSION = (1, 0)
  15 +
5 16 __all__ = [p for p in dir() if not p.startswith('_')]
6 17
10 gaspar/consumers.py
@@ -3,6 +3,7 @@
3 3
4 4 """Gaspar consumers (workers)."""
5 5
  6 +import os
6 7 import logging
7 8 import eventlet
8 9 from eventlet.green import zmq
@@ -26,6 +27,9 @@ def initialize(self, producer):
26 27 self.initialized = True
27 28
28 29 def start(self):
  30 + """Start the consumer. This starts a listen loop on a zmq.PULL socket,
  31 + calling ``self.handle`` on each incoming request and pushing the response
  32 + on a zmq.PUSH socket back to the producer."""
29 33 if not self.initialized:
30 34 raise Exception("Consumer not initialized (no Producer).")
31 35 producer = self.producer
@@ -38,7 +42,7 @@ def start(self):
38 42 self.listen()
39 43
40 44 def listen(self):
41   - import os
  45 + """Listen forever on the zmq.PULL socket."""
42 46 while True:
43 47 message = self.pull.recv()
44 48 logger.debug("received message of length %d" % len(message))
@@ -47,7 +51,9 @@ def listen(self):
47 51 self.push.send(response)
48 52
49 53 def handle(self, message):
50   - """Default handler, returns message."""
  54 + """Handle a message. If this producer was initialized with a handler,
  55 + that handler is called with ``message`` as an argument, and its return
  56 + value is sent over the zmq.PUSH socket back to the producer."""
51 57 if self.handler:
52 58 return self.handler(message)
53 59 return message
4 gaspar/producers.py
@@ -91,6 +91,10 @@ def serve(self):
91 91 eventlet.spawn(self.request_handler, conn, addr)
92 92
93 93 def start(self, blocking=True):
  94 + """Start the producer. This will eventually fire the ``server_start``
  95 + and ``running`` events in sequence, which signify that the incoming
  96 + TCP request socket is running and the workers have been forked,
  97 + respectively. If ``blocking`` is False, control ."""
94 98 self.setup_zmq()
95 99 if blocking:
96 100 self.serve()
11 setup.py
@@ -6,7 +6,11 @@
6 6 from setuptools import setup, find_packages
7 7 import sys, os
8 8
9   -version = '0.1'
  9 +try:
  10 + from gaspar import VERSION
  11 + version = '.'.join(VERSION)
  12 +except ImportError:
  13 + version = '1.0'
10 14
11 15 # some trove classifiers:
12 16
@@ -21,7 +25,10 @@
21 25 long_description=open('README.rst').read(),
22 26 # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
23 27 classifiers=[
24   - 'Development Status :: 1 - Planning',
  28 + 'Development Status :: 4 - Beta',
  29 + 'License :: OSI Approved :: MIT License',
  30 + 'Intended Audience :: Developers',
  31 + 'Operating System :: POSIX',
25 32 ],
26 33 keywords='eventlet zmq parallel prefork',
27 34 author='Jason Moiron',

0 comments on commit 3c75a45

Please sign in to comment.
Something went wrong with that request. Please try again.