0
@@ -42,11 +42,24 @@ static void
0
 conn_free(conn c)
0
 {
0
     c->fd = 0;
0
+    TUBE_ASSIGN(c->use, NULL);
0
     conn_insert(&pool, c);
0
 }
0
 
0
+static void
0
+inc(ms a, tube t, size_t i)
0
+{
0
+    tube_iref(t);
0
+}
0
+
0
+static void
0
+dec(ms a, tube t, size_t i)
0
+{
0
+    tube_dref(t);
0
+}
0
+
0
 conn
0
-make_conn(int fd, char start_state)
0
+make_conn(int fd, char start_state, tube use, tube watch)
0
 {
0
     job j;
0
     conn c;
0
@@ -54,6 +67,15 @@ make_conn(int fd, char start_state)
0
     c = conn_alloc();
0
     if (!c) return twarn("OOM"), NULL;
0
 
0
+    ms_init(&c->watch, (ms_event_fn) inc, (ms_event_fn) dec);
0
+    if (!ms_append(&c->watch, watch)) {
0
+        conn_free(c);
0
+        return twarn("OOM"), NULL;
0
+    }
0
+
0
+    c->use = NULL; /* initialize */
0
+    TUBE_ASSIGN(c->use, use);
0
+
0
     c->fd = fd;
0
     c->state = start_state;
0
     c->type = 0;
0
@@ -149,7 +171,7 @@ conn_update_evq(conn c, const int events)
0
     return conn_set_evq(c, events, c->evq.ev_callback);
0
 }
0
 
0
-int
0
+static int
0
 conn_list_any_p(conn head)
0
 {
0
     return head->next != head || head->prev != head;
0
@@ -209,10 +231,10 @@ conn_close(conn c)
0
 
0
     close(c->fd);
0
 
0
-    free(c->in_job);
0
+    job_free(c->in_job);
0
 
0
     /* was this a peek or stats command? */
0
-    if (!has_reserved_this_job(c, c->out_job)) free(c->out_job);
0
+    if (!has_reserved_this_job(c, c->out_job)) job_free(c->out_job);
0
 
0
     c->in_job = c->out_job = NULL;
0
 
0
@@ -226,5 +248,7 @@ conn_close(conn c)
0
     conn_remove(c);
0
     if (has_reserved_job(c)) enqueue_reserved_jobs(c);
0
 
0
+    ms_clear(&c->watch);
0
+
0
     conn_free(c);
0
 }

0
@@ -20,7 +20,9 @@
0
 #define conn_h
0
 
0
 #include "event.h"
0
+#include "ms.h"
0
 #include "job.h"
0
+#include "tube.h"
0
 
0
 #define STATE_WANTCOMMAND 0
0
 #define STATE_WANTDATA 1
0
@@ -44,6 +46,9 @@
0
 #define OP_STATS 8
0
 #define OP_JOBSTATS 9
0
 #define OP_PEEK 10
0
+#define OP_USE 11
0
+#define OP_WATCH 12
0
+#define OP_IGNORE 13
0
 
0
 /* CONN_TYPE_* are bit masks */
0
 #define CONN_TYPE_PRODUCER 1
0
@@ -74,16 +79,17 @@ struct conn {
0
     job out_job;
0
     int out_job_sent;
0
     struct job reserved_jobs; /* doubly-linked list header */
0
+    tube use;
0
+    struct ms watch;
0
 };
0
 
0
-conn make_conn(int fd, char start_state);
0
+conn make_conn(int fd, char start_state, tube use, tube watch);
0
 
0
 int conn_set_evq(conn c, const int events, evh handler);
0
 int conn_update_evq(conn c, const int flags);
0
 
0
 void conn_close(conn c);
0
 
0
-int conn_list_any_p(conn head);
0
 conn conn_remove(conn c);
0
 void conn_insert(conn head, conn c);
0
 

0
@@ -10,6 +10,12 @@ were received and sends responses in the same order. All integers in the
0
 protocol are formatted in decimal and (unless otherwise indicated)
0
 nonnegative.
0
 
0
+Names, in this protocol, are ASCII strings. They may contain letters (A-Z and
0
+a-z), numerals (0-9), hyphen ("-"), plus ("+"), slash ("/"), semicolon (";"),
0
+dot ("."), dollar-sign ("$"), and parentheses ("(" and ")"), but they may not
0
+begin with a hyphen. They are terminated by white space (either a space char or
0
+end of line). Each name must be at least one character long.
0
+
0
 The protocol contains two kinds of data: text lines and unstructured chunks of
0
 data. Text lines are used for client commands and server responses. Chunks are
0
 used to transfer job bodies and stats information. Each job body is an opaque
0
@@ -106,9 +112,23 @@ Here is a picture with more possibilities:
0
                         `--------> *poof*
0
 
0
 
0
+The system has one or more tubes. Each tube consists of a ready queue and a
0
+delay queue. Each job spends its entire life in one tube. Consumers can show
0
+interest in tubes by sending the "watch" command; they can show disinterest by
0
+sending the "ignore" command. This set of interesting tubes is said to be a
0
+consumer's "watch list". When a client reserves a job, it may come from any of
0
+the tubes in its watch list.
0
+
0
+When a client connects, its watch list is initially just the tube named
0
+"default". If it submits jobs without having sent a "use" command, they will
0
+live in the tube named "default".
0
+
0
+Tubes are created on demand whenever they are referenced. If a tube is empty
0
+(that is, it contains no ready, delayed, or buried jobs) and no client refers
0
+to it, it will be deleted.
0
 
0
-Producer Command
0
-----------------
0
+Producer Commands
0
+-----------------
0
 
0
 The "put" command is for any process that wants to insert a job into the queue.
0
 It comprises a command line followed by the job body:
0
@@ -116,7 +136,8 @@ It comprises a command line followed by the job body:
0
 put <pri> <delay> <ttr> <bytes>\r\n
0
 <data>\r\n
0
 
0
-It inserts a job into the queue.
0
+It inserts a job into the client's currently used tube (see the "use" command
0
+below).
0
 
0
  - <pri> is an integer < 2**32. Jobs with smaller priority values will be
0
    scheduled before jobs with larger priorities. The most urgent priority is 0;
0
@@ -150,6 +171,19 @@ may be:
0
 
0
    - <id> is the integer id of the new job
0
 
0
+The "use" command is for producers. Subsequent put commands will put jobs into
0
+the tube specified by this command. If no use command has been issued, jobs
0
+will be put into the tube named "default".
0
+
0
+use <tube>\r\n
0
+
0
+ - <tube> is a name at most 200 bytes. It specifies the tube to use. If the
0
+   tube does not exist, it will be created.
0
+
0
+The only reply is:
0
+
0
+USING\r\n
0
+
0
 Worker Commands
0
 ---------------
0
 
0
@@ -239,6 +273,36 @@ There are two possible responses:
0
 
0
  - "NOT_FOUND\r\n" if the job does not exist or is not reserved by the client.
0
 
0
+The "watch" command adds the named tube to the watch list for the current
0
+connection. A reserve command will take a job from any of the tubes in the
0
+watch list. For each new connection, the watch list initially consists of one
0
+tube, named "default".
0
+
0
+watch <tube>\r\n
0
+
0
+ - <tube> is a name at most 200 bytes. It specifies a tube to add to the watch
0
+   list.
0
+
0
+The reply is:
0
+
0
+WATCHING <count>\r\n
0
+
0
+ - <count> is the integer number of tubes currently in the watch list.
0
+
0
+The "ignore" command is for consumers. It removes the named tube from the
0
+watch list for the current connection.
0
+
0
+ignore <tube>\r\n
0
+
0
+The reply is one of:
0
+
0
+ - "WATCHING <count>\r\n" to indicate success.
0
+
0
+   - <count> is the integer number of tubes currently in the watch list.
0
+
0
+ - "NOT_IGNORED\r\n" if the client attempts to ignore the only tube in its
0
+   watch list.
0
+
0
 Other Commands
0
 --------------
0
 
0
@@ -315,6 +379,8 @@ scalars.
0
 
0
    - "id" is the job id
0
 
0
+   - "tube" is the name of the tube that contains this job
0
+
0
    - "state" is "ready" or "delayed" or "reserved" or "buried"
0
 
0
    - "age" is the time in seconds since the put command that created this job.
0
@@ -366,6 +432,8 @@ scalars.
0
 
0
    - "total-jobs" is the cumulative count of jobs created.
0
 
0
+   - "current-tubes" is the number of currently-existing tubes.
0
+
0
    - "current-connections" is the number of currently open connections.
0
 
0
    - "current-producers" is the number of open connections that have each

0
@@ -20,6 +20,7 @@
0
 #include <string.h>
0
 
0
 #include "job.h"
0
+#include "tube.h"
0
 #include "util.h"
0
 
0
 static unsigned long long int next_id = 1;
0
@@ -37,12 +38,14 @@ allocate_job(int body_size)
0
     j->timeout_ct = j->release_ct = j->bury_ct = j->kick_ct = 0;
0
     j->body_size = body_size;
0
     j->next = j->prev = j; /* not in a linked list */
0
+    j->tube = NULL;
0
 
0
     return j;
0
 }
0
 
0
 job
0
-make_job(unsigned int pri, unsigned int delay, unsigned int ttr, int body_size)
0
+make_job(unsigned int pri, unsigned int delay, unsigned int ttr, int body_size,
0
+         tube tube)
0
 {
0
     job j;
0
 
0
@@ -53,10 +56,18 @@ make_job(unsigned int pri, unsigned int delay, unsigned int ttr, int body_size)
0
     j->pri = pri;
0
     j->delay = delay;
0
     j->ttr = ttr;
0
+    TUBE_ASSIGN(j->tube, tube);
0
 
0
     return j;
0
 }
0
 
0
+void
0
+job_free(job j)
0
+{
0
+    if (j) TUBE_ASSIGN(j->tube, NULL);
0
+    free(j);
0
+}
0
+
0
 int
0
 job_pri_cmp(job a, job b)
0
 {
0
@@ -91,10 +102,9 @@ job_copy(job j)
0
     n = malloc(sizeof(struct job) + j->body_size);
0
     if (!n) return twarnx("OOM"), NULL;
0
 
0
-    n->id = j->id;
0
-    n->pri = j->pri;
0
-    n->body_size = j->body_size;
0
-    memcpy(n->body, j->body, j->body_size);
0
+    memcpy(n, j, sizeof(struct job) + j->body_size);
0
+    n->next = n->prev = n; /* not in a linked list */
0
+    TUBE_ASSIGN(n->tube, j->tube);
0
 
0
     return n;
0
 }

0
@@ -21,14 +21,17 @@
0
 
0
 #include <time.h>
0
 
0
+typedef struct job *job;
0
+typedef int(*job_cmp_fn)(job, job);
0
+
0
+#include "tube.h"
0
+
0
 #define JOB_STATE_INVALID 0
0
 #define JOB_STATE_READY 1
0
 #define JOB_STATE_RESERVED 2
0
 #define JOB_STATE_BURIED 3
0
 #define JOB_STATE_DELAYED 4
0
 
0
-typedef struct job *job;
0
-
0
 struct job {
0
     job prev, next; /* linked list of jobs */
0
     unsigned long long int id;
0
@@ -42,15 +45,16 @@ struct job {
0
     unsigned int release_ct;
0
     unsigned int bury_ct;
0
     unsigned int kick_ct;
0
+    tube tube;
0
     char state;
0
     char body[];
0
 };
0
 
0
 job allocate_job(int body_size);
0
 job make_job(unsigned int pri, unsigned int delay, unsigned int ttr,
0
-        int body_size);
0
+             int body_size, tube tube);
0
+void job_free(job j);
0
 
0
-typedef int(*job_cmp_fn)(job, job);
0
 int job_pri_cmp(job a, job b);
0
 int job_delay_cmp(job a, job b);
0
 

0
@@ -20,35 +20,39 @@
0
 #include <stdio.h>
0
 #include <string.h>
0
 
0
+#include "tube.h" /* hack to make cpp happy */
0
 #include "pq.h"
0
 
0
-pq
0
-make_pq(unsigned int initial_cap, job_cmp_fn cmp)
0
+void
0
+pq_init(pq q, job_cmp_fn cmp)
0
 {
0
-    pq q;
0
+    if (!q) return;
0
 
0
-    q = malloc(sizeof(struct pq));
0
-    if (!q) return NULL;
0
-
0
-    q->cap = initial_cap;
0
+    q->cap = 0;
0
     q->used = 0;
0
     q->cmp = cmp;
0
-    q->heap = malloc(initial_cap * sizeof(job));
0
-    if (!q->heap) return free(q), NULL;
0
+    q->heap = NULL;
0
+
0
+    return;
0
+}
0
 
0
-    return q;
0
+void
0
+pq_clear(pq q)
0
+{
0
+    free(q->heap);
0
+    pq_init(q, q->cmp);
0
 }
0
 
0
 static void
0
 pq_grow(pq q)
0
 {
0
     job *nheap;
0
-    unsigned int ncap = q->cap << 1;
0
+    unsigned int ncap = q->cap << 1 ? : 1;
0
 
0
     nheap = malloc(ncap * sizeof(job));
0
     if (!nheap) return;
0
 
0
-    memcpy(nheap, q->heap, q->used * sizeof(job));
0
+    if (q->heap) memcpy(nheap, q->heap, q->used * sizeof(job));
0
     free(q->heap);
0
     q->heap = nheap;
0
     q->cap = ncap;

0
@@ -19,17 +19,21 @@
0
 #ifndef q_h
0
 #define q_h
0
 
0
+typedef struct pq *pq;
0
+
0
 #include "job.h"
0
 
0
-typedef struct pq {
0
+struct pq {
0
     unsigned int cap;
0
     unsigned int used;
0
     job_cmp_fn cmp;
0
     job *heap;
0
-} *pq;
0
+};
0
+
0
+/* initialize a priority queue */
0
+void pq_init(pq q, job_cmp_fn cmp);
0
 
0
-/* make a priority queue with initial capacity initial_cap */
0
-pq make_pq(unsigned int initial_cap, job_cmp_fn cmp);
0
+void pq_clear(pq q);
0
 
0
 /* return 1 if the job was inserted, else 0 */
0
 int pq_give(pq q, job j);

0
@@ -27,7 +27,9 @@
0
 
0
 #include "prot.h"
0
 #include "pq.h"
0
+#include "ms.h"
0
 #include "job.h"
0
+#include "tube.h"
0
 #include "conn.h"
0
 #include "util.h"
0
 #include "net.h"
0
@@ -36,6 +38,11 @@
0
 /* job body cannot be greater than this many bytes long */
0
 #define JOB_DATA_SIZE_LIMIT ((1 << 16) - 1)
0
 
0
+#define NAME_CHARS \
0
+    "ABCDEFGHIJKLMNOPQRSTUVWXYZ" \
0
+    "abcdefghijklmnopqrstuvwxyz" \
0
+    "0123456789-+/;.$()"
0
+
0
 #define CMD_PUT "put "
0
 #define CMD_PEEK "peek"
0
 #define CMD_PEEKJOB "peek "
0
@@ -46,6 +53,9 @@
0
 #define CMD_KICK "kick "
0
 #define CMD_STATS "stats"
0
 #define CMD_JOBSTATS "stats "
0
+#define CMD_USE "use "
0
+#define CMD_WATCH "watch "
0
+#define CMD_IGNORE "ignore "
0
 
0
 #define CONSTSTRLEN(m) (sizeof(m) - 1)
0
 
0
@@ -58,6 +68,9 @@
0
 #define CMD_KICK_LEN CONSTSTRLEN(CMD_KICK)
0
 #define CMD_STATS_LEN CONSTSTRLEN(CMD_STATS)
0
 #define CMD_JOBSTATS_LEN CONSTSTRLEN(CMD_JOBSTATS)
0
+#define CMD_USE_LEN CONSTSTRLEN(CMD_USE)
0
+#define CMD_WATCH_LEN CONSTSTRLEN(CMD_WATCH)
0
+#define CMD_IGNORE_LEN CONSTSTRLEN(CMD_IGNORE)
0
 
0
 #define MSG_FOUND "FOUND"
0
 #define MSG_NOTFOUND "NOT_FOUND\r\n"
0
@@ -67,11 +80,13 @@
0
 #define MSG_BURIED "BURIED\r\n"
0
 #define MSG_BURIED_FMT "BURIED %llu\r\n"
0
 #define MSG_INSERTED_FMT "INSERTED %llu\r\n"
0
+#define MSG_NOT_IGNORED "NOT_IGNORED\r\n"
0
 
0
 #define MSG_NOTFOUND_LEN CONSTSTRLEN(MSG_NOTFOUND)
0
 #define MSG_DELETED_LEN CONSTSTRLEN(MSG_DELETED)
0
 #define MSG_RELEASED_LEN CONSTSTRLEN(MSG_RELEASED)
0
 #define MSG_BURIED_LEN CONSTSTRLEN(MSG_BURIED)
0
+#define MSG_NOT_IGNORED_LEN CONSTSTRLEN(MSG_NOT_IGNORED)
0
 
0
 #define MSG_OUT_OF_MEMORY "OUT_OF_MEMORY\r\n"
0
 #define MSG_INTERNAL_ERROR "INTERNAL_ERROR\r\n"
0
@@ -97,6 +112,7 @@
0
     "cmd-stats: %llu\n" \
0
     "job-timeouts: %llu\n" \
0
     "total-jobs: %llu\n" \
0
+    "current-tubes: %u\n" \
0
     "current-connections: %u\n" \
0
     "current-producers: %u\n" \
0
     "current-workers: %u\n" \